diff --git "a/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" "b/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" @@ -0,0 +1,5209 @@ +2024-03-27 19:05:13,649 INFO StreamThr :1895319 [internal.py:wandb_internal():86] W&B internal server running at pid: 1895319, started at: 2024-03-27 19:05:13.648580 +2024-03-27 19:05:13,651 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status +2024-03-27 19:05:13,653 INFO WriterThread:1895319 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/run-7p2x8a0l.wandb +2024-03-27 19:05:13,654 DEBUG SenderThread:1895319 [sender.py:send():382] send: header +2024-03-27 19:05:13,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: run +2024-03-27 19:05:14,021 INFO SenderThread:1895319 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files +2024-03-27 19:05:14,021 INFO SenderThread:1895319 [sender.py:_start_run_threads():1136] run started: 7p2x8a0l with start time 1711562713.647808 +2024-03-27 19:05:14,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 19:05:14,029 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: check_version +2024-03-27 19:05:14,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 19:05:14,643 DEBUG HandlerThread:1895319 [system_info.py:__init__():27] System info init +2024-03-27 19:05:14,643 DEBUG HandlerThread:1895319 [system_info.py:__init__():42] System info init done +2024-03-27 19:05:14,643 INFO HandlerThread:1895319 [system_monitor.py:start():194] Starting system monitor +2024-03-27 19:05:14,643 INFO SystemMonitor:1895319 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 19:05:14,644 INFO HandlerThread:1895319 [system_monitor.py:probe():214] Collecting system info +2024-03-27 19:05:14,645 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 19:05:14,646 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started disk monitoring +2024-03-27 19:05:14,647 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 19:05:14,648 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started memory monitoring +2024-03-27 19:05:14,650 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started network monitoring +2024-03-27 19:05:14,699 DEBUG HandlerThread:1895319 [system_info.py:probe():151] Probing system +2024-03-27 19:05:14,705 DEBUG HandlerThread:1895319 [system_info.py:_probe_git():136] Probing git +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_info.py:_probe_git():144] Probing git done +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_info.py:probe():199] Probing system done +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T18:05:14.699269', 'startedAt': '2024-03-27T18:05:13.643873', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--save_total_limit=1', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--freeze_encoder', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft-frozen-encoder', 'commit': 'e7946df277d73ac75c34c2017b01c6d39e0275cd'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft-frozen-encoder', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2055.302406250001, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 1785.051, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.027, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.844, 'min': 1500.0, 'max': 2250.0}, {'current': 2279.976, 'min': 1500.0, 'max': 2250.0}, {'current': 3328.296, 'min': 1500.0, 'max': 2250.0}, {'current': 2778.772, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.514, 'min': 1500.0, 'max': 2250.0}, {'current': 1679.17, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.11, 'min': 1500.0, 'max': 2250.0}, {'current': 3342.154, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.65, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.492, 'min': 1500.0, 'max': 2250.0}, {'current': 3344.267, 'min': 1500.0, 'max': 2250.0}, {'current': 1991.129, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.832, 'min': 1500.0, 'max': 2250.0}, {'current': 1681.287, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.975, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.691, 'min': 1500.0, 'max': 2250.0}, {'current': 3343.923, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.613, 'min': 1500.0, 'max': 2250.0}, {'current': 1711.757, 'min': 1500.0, 'max': 2250.0}, {'current': 2605.829, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.095, 'min': 1500.0, 'max': 2250.0}, {'current': 1868.607, 'min': 1500.0, 'max': 2250.0}, {'current': 1761.225, 'min': 1500.0, 'max': 2250.0}, {'current': 1722.608, 'min': 1500.0, 'max': 2250.0}, {'current': 2446.611, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.419, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.485, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.608, 'min': 1500.0, 'max': 2250.0}, {'current': 1774.005, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.879, 'min': 1500.0, 'max': 2250.0}, {'current': 3356.822, 'min': 1500.0, 'max': 2250.0}, {'current': 1674.163, 'min': 1500.0, 'max': 2250.0}, {'current': 1676.528, 'min': 1500.0, 'max': 2250.0}, {'current': 3346.209, 'min': 1500.0, 'max': 2250.0}, {'current': 1679.409, 'min': 1500.0, 'max': 2250.0}, {'current': 1673.8, 'min': 1500.0, 'max': 2250.0}, {'current': 1675.44, 'min': 1500.0, 'max': 2250.0}, {'current': 1722.367, 'min': 1500.0, 'max': 2250.0}, {'current': 1721.361, 'min': 1500.0, 'max': 2250.0}, {'current': 1719.425, 'min': 1500.0, 'max': 2250.0}, {'current': 2130.252, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.838, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.535, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.69, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.257, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.417, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.058, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.79, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.254, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.759, 'min': 1500.0, 'max': 2250.0}, {'current': 3285.802, 'min': 1500.0, 'max': 2250.0}, {'current': 3327.549, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.454, 'min': 1500.0, 'max': 2250.0}, {'current': 2291.123, 'min': 1500.0, 'max': 2250.0}, {'current': 1845.287, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.335, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.798, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.761, 'min': 1500.0, 'max': 2250.0}, {'current': 2172.943, 'min': 1500.0, 'max': 2250.0}, {'current': 2109.384, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.332, 'min': 1500.0, 'max': 2250.0}, {'current': 2173.511, 'min': 1500.0, 'max': 2250.0}, {'current': 2187.364, 'min': 1500.0, 'max': 2250.0}, {'current': 2152.119, 'min': 1500.0, 'max': 2250.0}, {'current': 3315.314, 'min': 1500.0, 'max': 2250.0}, {'current': 3333.811, 'min': 1500.0, 'max': 2250.0}, {'current': 1966.133, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.353, 'min': 1500.0, 'max': 2250.0}, {'current': 2183.755, 'min': 1500.0, 'max': 2250.0}, {'current': 1836.471, 'min': 1500.0, 'max': 2250.0}, {'current': 3319.659, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.1, 'min': 1500.0, 'max': 2250.0}, {'current': 3156.296, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.519, 'min': 1500.0, 'max': 2250.0}, {'current': 3259.492, 'min': 1500.0, 'max': 2250.0}, {'current': 2267.628, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.441, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.329, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.974, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.269, 'min': 1500.0, 'max': 2250.0}, {'current': 3315.801, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.948, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.171, 'min': 1500.0, 'max': 2250.0}, {'current': 3147.217, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.184, 'min': 1500.0, 'max': 2250.0}, {'current': 2145.9, 'min': 1500.0, 'max': 2250.0}, {'current': 2302.183, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1812.149, 'min': 1500.0, 'max': 2250.0}, {'current': 1739.416, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.942, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.725, 'min': 1500.0, 'max': 2250.0}, {'current': 1846.358, 'min': 1500.0, 'max': 2250.0}, {'current': 3324.686, 'min': 1500.0, 'max': 2250.0}, {'current': 3313.397, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.908, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.332, 'min': 1500.0, 'max': 2250.0}, {'current': 3357.779, 'min': 1500.0, 'max': 2250.0}, {'current': 2196.764, 'min': 1500.0, 'max': 2250.0}, {'current': 1629.818, 'min': 1500.0, 'max': 2250.0}, {'current': 1548.482, 'min': 1500.0, 'max': 2250.0}, {'current': 1550.408, 'min': 1500.0, 'max': 2250.0}, {'current': 1551.062, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.314, 'min': 1500.0, 'max': 2250.0}, {'current': 2797.062, 'min': 1500.0, 'max': 2250.0}, {'current': 1556.743, 'min': 1500.0, 'max': 2250.0}, {'current': 2040.048, 'min': 1500.0, 'max': 2250.0}, {'current': 1557.951, 'min': 1500.0, 'max': 2250.0}, {'current': 1548.578, 'min': 1500.0, 'max': 2250.0}, {'current': 2204.931, 'min': 1500.0, 'max': 2250.0}, {'current': 2191.05, 'min': 1500.0, 'max': 2250.0}, {'current': 2206.083, 'min': 1500.0, 'max': 2250.0}, {'current': 2179.071, 'min': 1500.0, 'max': 2250.0}, {'current': 2137.078, 'min': 1500.0, 'max': 2250.0}, {'current': 2908.115, 'min': 1500.0, 'max': 2250.0}, {'current': 3329.978, 'min': 1500.0, 'max': 2250.0}, {'current': 2146.791, 'min': 1500.0, 'max': 2250.0}, {'current': 1835.104, 'min': 1500.0, 'max': 2250.0}, {'current': 3279.409, 'min': 1500.0, 'max': 2250.0}, {'current': 2340.495, 'min': 1500.0, 'max': 2250.0}, {'current': 2350.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1860.834, 'min': 1500.0, 'max': 2250.0}, {'current': 1928.589, 'min': 1500.0, 'max': 2250.0}, {'current': 3299.491, 'min': 1500.0, 'max': 2250.0}, {'current': 1897.289, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1610.3510513305664}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 19:05:14,716 INFO HandlerThread:1895319 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 19:05:14,716 INFO HandlerThread:1895319 [system_monitor.py:probe():227] Publishing system info +2024-03-27 19:05:14,718 INFO HandlerThread:1895319 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 19:05:14,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: files +2024-03-27 19:05:14,722 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 19:05:14,726 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 19:05:14,727 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: python_packages +2024-03-27 19:05:14,728 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:14,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:14,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:14,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,960 DEBUG SenderThread:1895319 [sender.py:send():382] send: config +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:05:14,961 WARNING SenderThread:1895319 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/requirements.txt +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-metadata.json +2024-03-27 19:05:15,268 INFO wandb-upload_0:1895319 [upload_job.py:push():131] Uploaded file /tmp/tmpdusptjiswandb/d02ankfh-wandb-metadata.json +2024-03-27 19:05:16,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:17,022 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:18,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:19,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:19,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,793 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:23,963 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:24,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:24,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,752 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,026 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:29,424 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:29,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:31,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:32,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:32,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:35,001 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:35,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:35,028 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:35,074 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:37,029 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:37,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:37,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:39,029 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:40,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,405 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:41,030 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:42,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:42,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,030 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:45,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:45,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:45,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:46,250 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:47,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:47,032 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:05:47,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:47,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:49,032 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:50,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:50,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:51,033 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:51,857 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:53,033 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:53,896 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:05:53,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:53,931 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,034 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:55,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:57,035 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:57,850 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:58,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:58,826 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:59,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:59,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:00,036 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:00,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:02,036 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:03,461 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:03,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:03,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:04,037 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:05,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:05,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:06,038 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:08,038 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:08,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:09,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:10,039 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:10,114 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:10,116 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:10,117 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:11,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:11,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:11,039 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:06:12,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:13,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:13,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:13,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:14,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:14,426 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:14,651 DEBUG SystemMonitor:1895319 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 19:06:14,653 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:06:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:14,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:15,938 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:15,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:16,041 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:18,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:18,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:18,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:20,040 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:20,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:20,993 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:21,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:21,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:06:22,043 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:23,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:23,892 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:24,043 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:25,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:25,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:26,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:26,044 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:28,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:28,922 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:28,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:29,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:30,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:31,652 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:32,046 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:32,505 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:32,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,537 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:34,047 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:35,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:35,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:36,047 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:37,625 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:37,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:38,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:40,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:40,649 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:40,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,049 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:42,699 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,708 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:43,246 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:44,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:44,655 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:06:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:44,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:45,552 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:45,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:46,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:47,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:48,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:49,216 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:50,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:50,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:50,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:51,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:51,556 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:06:51,557 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:51,557 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:52,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:06:52,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:52,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:52,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:53,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:54,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:54,301 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:55,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:55,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:56,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:57,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:57,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:58,054 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:59,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:59,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:59,923 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:00,054 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:00,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:00,814 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:02,055 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:02,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:02,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,055 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:05,869 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:06,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:07,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:07,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:08,057 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:10,057 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:11,497 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:11,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,526 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,539 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:12,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:14,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:14,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,532 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,657 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:07:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:14,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:16,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:16,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:16,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:17,498 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:18,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:19,608 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:20,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:21,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:21,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:22,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:23,113 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:24,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:24,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:24,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:26,546 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:28,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:29,072 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:29,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:29,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:29,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:30,063 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:31,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:31,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:32,063 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:33,067 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:07:33,069 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:07:33,070 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:07:33,070 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:07:34,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:07:34,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:34,587 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,724 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:35,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:36,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:36,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:36,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:38,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:39,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,671 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:40,066 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:40,699 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:41,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:41,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:42,067 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:44,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:44,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,658 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:07:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:45,861 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:46,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:46,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:46,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:48,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:50,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:50,459 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:51,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:51,299 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:51,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:52,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:53,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:53,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:54,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:55,360 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:56,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:57,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:58,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:58,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:58,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:00,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:00,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,165 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:02,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:02,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:03,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:03,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:04,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:05,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:05,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:06,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:08,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:08,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,221 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,902 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:10,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:10,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:10,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:12,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:13,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:14,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:14,431 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:14,435 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:14,437 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:08:14,437 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:14,437 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:14,660 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:08:14,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:14,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:15,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:08:15,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:15,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:17,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:18,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:18,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:20,111 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:22,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:23,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,125 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:24,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:25,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:25,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:25,860 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:26,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:28,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:28,670 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:28,690 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:30,081 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:31,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,823 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:32,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:33,612 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:34,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:36,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:36,527 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:36,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:37,465 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:38,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:38,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,592 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:40,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:41,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:41,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:42,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:43,442 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:43,478 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:44,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:44,662 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:08:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:44,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:46,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:46,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:46,472 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:48,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:48,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:48,503 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:49,096 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:50,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:50,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:52,088 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:53,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:53,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,088 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:55,075 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:55,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:55,754 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:08:55,755 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:55,755 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:56,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:08:56,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:57,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:58,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:58,480 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,492 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:59,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:00,090 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:00,511 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:00,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:00,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:02,091 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:03,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,658 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:04,092 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:06,093 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:06,694 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:07,258 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:07,276 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:07,290 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,094 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:10,094 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:10,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:10,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,095 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:12,199 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,342 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:14,096 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:14,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:14,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:14,663 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:09:14,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:14,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:16,097 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:16,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:17,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:18,097 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:18,331 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:19,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:19,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:20,098 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:22,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,098 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:23,479 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:24,099 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:24,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:24,110 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:26,100 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:27,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:28,100 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:29,120 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:29,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:29,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:30,101 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:32,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,102 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:32,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,102 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:34,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:35,115 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:36,103 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:37,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,305 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,465 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:09:37,466 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:09:37,467 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:09:37,467 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:09:38,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:09:38,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:39,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:39,329 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:39,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:40,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:40,131 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:42,105 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:42,188 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:42,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:44,105 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:44,664 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:09:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:44,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:44,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:45,769 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:45,779 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:45,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:45,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:46,106 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:47,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:48,107 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:48,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:50,107 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:50,781 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:50,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:50,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,108 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:52,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:54,109 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:55,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:55,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,109 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:56,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:57,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:57,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:58,110 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:00,110 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:00,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:00,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:02,111 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:02,408 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:02,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:02,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:04,111 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:04,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:04,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:06,112 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:07,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:07,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,113 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:08,395 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:10,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,113 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:12,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,114 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:14,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:14,115 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:14,666 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:10:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:14,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:14,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:16,116 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:16,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:16,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,116 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:18,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,870 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:10:18,872 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:10:18,872 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:10:18,872 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:10:19,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:10:19,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:20,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:21,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:22,118 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:22,132 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:22,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:22,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:24,118 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:25,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,511 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:26,119 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:27,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:27,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:28,120 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:29,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:29,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:30,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,120 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:31,481 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:32,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:32,121 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:32,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,122 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:34,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:36,122 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:37,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:37,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:37,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:38,123 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:39,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:39,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,124 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:41,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:41,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,124 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:42,286 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:43,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:43,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:44,125 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:44,668 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:10:44,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:44,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:45,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:46,126 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:48,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,126 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:48,271 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:50,127 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:50,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:50,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,127 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:52,811 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:53,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:54,128 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:54,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,129 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:58,130 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:58,264 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:58,291 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:58,309 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:58,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:59,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:59,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:59,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:00,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:00,582 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:00,584 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:11:00,584 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:00,585 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:01,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:11:02,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:03,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:03,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:03,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:04,132 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:04,140 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:05,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:05,923 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:06,132 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:07,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:07,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:08,133 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:09,302 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:09,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:09,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:10,134 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:12,134 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:13,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:13,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:14,135 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:14,362 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:14,669 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:11:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:15,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:15,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,135 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:18,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:18,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:18,136 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:20,137 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:20,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:20,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:20,330 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:22,138 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:23,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:23,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:24,138 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:25,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:25,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:26,321 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:28,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:28,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:28,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:29,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:29,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:30,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:30,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:30,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:31,531 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:32,141 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:33,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:33,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:34,141 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:36,142 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:37,001 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:37,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:37,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:37,496 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:38,143 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:39,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:40,144 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:41,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:41,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:43,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:44,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,671 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:11:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:46,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:48,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:52,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:53,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:53,065 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:54,091 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:54,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:54,773 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:54,774 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:11:54,774 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:54,775 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:55,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:11:55,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:55,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:56,149 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:57,149 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:57,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:57,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:58,150 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:59,402 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:59,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:59,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:00,150 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:00,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:00,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:02,151 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:03,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:03,020 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:04,151 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:05,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:06,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,152 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:06,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,152 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:08,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:10,153 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:11,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:11,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:11,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:12,154 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:14,155 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:14,672 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:12:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:14,874 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:14,894 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:14,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,155 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:16,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:17,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:17,916 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:18,157 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:19,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:19,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:20,157 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:22,158 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:22,366 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:22,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:22,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:24,158 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:25,004 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:25,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,159 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:27,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,015 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:28,160 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:29,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:29,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:30,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:30,037 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:30,160 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:32,161 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:33,037 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:34,006 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:34,161 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:35,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:35,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:36,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:36,327 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:12:36,329 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:12:36,329 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:12:36,330 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:12:37,125 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,142 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:12:38,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:39,163 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:39,640 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:40,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,163 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:42,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,165 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:44,165 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:44,674 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:12:44,677 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:44,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,930 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,166 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:46,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:47,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:48,167 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:49,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,939 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:50,120 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:50,167 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:52,168 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:53,525 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:53,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:53,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:54,169 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:55,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:55,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:56,108 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:56,169 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:58,170 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:58,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:58,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:59,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:59,731 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:00,171 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:00,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:00,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:01,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:02,172 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:03,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:03,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:04,173 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:05,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:05,434 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:06,174 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:07,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:08,174 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:08,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:08,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,175 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:10,569 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:12,176 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:13,407 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:13,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:13,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:14,176 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:14,675 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:13:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:15,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:16,177 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:17,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:18,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:18,388 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:18,390 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:13:18,390 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:18,391 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:19,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:13:19,391 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:20,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:20,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:20,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:21,179 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:22,179 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:22,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,537 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:24,180 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:25,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:25,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:25,434 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:26,181 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:27,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:28,181 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:29,731 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:30,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:30,182 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:31,082 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:31,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:31,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,183 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:33,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:33,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:34,183 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:35,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:35,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:36,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:36,184 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:37,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:37,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:38,185 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:40,185 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:40,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:40,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:41,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:42,188 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:42,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:44,190 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:44,677 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:13:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:44,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:45,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:45,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:46,191 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:46,398 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:47,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:47,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:48,191 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:50,192 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:50,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:50,844 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:51,399 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:52,193 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:52,862 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:52,875 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,194 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:55,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:55,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:56,194 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:57,377 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:57,801 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:57,814 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:58,195 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:59,684 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:59,686 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:13:59,686 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:59,686 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:59,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:59,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:00,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:14:00,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:00,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:00,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:01,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:02,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:02,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:02,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:03,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:04,197 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:05,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:05,721 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:06,198 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:08,198 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:08,990 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:09,298 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:09,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:10,199 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:11,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:11,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:12,199 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:14,107 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,200 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:14,636 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:14,678 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:14:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:14,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:16,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:16,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:16,201 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:18,201 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:19,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:19,217 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:20,202 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:20,620 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:21,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:21,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:22,207 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:24,207 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:24,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,255 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,208 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:26,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:26,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:28,209 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:29,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:29,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:29,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:30,209 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:31,289 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:31,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:31,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:32,210 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:34,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,210 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:36,211 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:36,218 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:36,227 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:37,273 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:38,211 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:39,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:39,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,212 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:41,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:41,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:41,284 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:14:41,285 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:14:41,286 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:14:41,286 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:14:42,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:14:42,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:42,287 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:43,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:44,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:44,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:44,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:44,680 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:14:44,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:44,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:44,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:46,214 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:47,291 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:47,867 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:47,885 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:48,215 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:50,216 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:50,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:51,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:52,216 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:52,947 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:53,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:53,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:54,217 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:56,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,218 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:57,952 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:58,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,115 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,218 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:59,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:00,219 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:00,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:00,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:02,220 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:03,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:04,221 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:05,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:05,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:06,222 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:07,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:07,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:08,222 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:09,594 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:10,223 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:10,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:10,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:12,224 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:12,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:12,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:14,225 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:14,681 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:15:14,684 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:14,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:15,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:15,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,225 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:17,843 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:17,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:18,226 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:20,226 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:20,571 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:20,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:20,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,227 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:22,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,922 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:15:22,923 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:15:22,924 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:15:22,924 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:15:23,227 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:15:24,228 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:25,228 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:26,188 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:26,218 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:26,226 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:26,229 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:26,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,229 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:28,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:30,230 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:31,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:32,216 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:32,230 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:33,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:33,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:34,231 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:35,765 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:35,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:36,232 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:37,795 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,867 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:38,232 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:40,233 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:40,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:40,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:42,233 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:42,696 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:42,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:43,848 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:44,234 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:44,683 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:15:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:45,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:45,752 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:46,235 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:47,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:48,235 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:49,507 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:50,236 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:50,794 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:50,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,237 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:52,827 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:54,238 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:55,385 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:55,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:55,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:56,239 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:57,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:57,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:58,240 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:59,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:00,240 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:00,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:00,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:00,964 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:02,241 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:04,242 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:04,406 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:16:04,408 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:16:04,408 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:16:04,409 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:16:04,604 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:04,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,634 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:05,242 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:16:06,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:06,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:06,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:07,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:07,460 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:08,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:09,480 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:09,489 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:10,244 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:11,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:11,518 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:12,048 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:12,244 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:14,245 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:14,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,685 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:16:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:14,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:16,246 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:16,498 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:16,507 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:17,701 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:18,246 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:19,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:20,247 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:21,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:21,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:22,248 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:23,687 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:24,249 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:24,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:24,403 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,250 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:26,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:28,250 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:29,348 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:29,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:29,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:30,251 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:31,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:32,252 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:34,252 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:34,609 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:34,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:35,328 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:36,253 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:36,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:38,254 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:39,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:39,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:40,255 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:40,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:42,255 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:43,258 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:43,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:44,256 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:44,687 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:16:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:45,988 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:46,001 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:16:46,002 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:16:46,003 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:16:46,003 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:16:46,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,257 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:16:46,257 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:47,258 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:48,258 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:48,262 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:48,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:50,259 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:50,530 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,366 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:52,260 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:53,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:53,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:54,260 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:56,261 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:56,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:57,622 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:58,261 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:58,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:58,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:59,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:00,262 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:01,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:02,263 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:03,408 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:03,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:03,593 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:04,263 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:06,264 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:06,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,264 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:08,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,599 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:10,265 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:11,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:12,265 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:13,609 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:13,675 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:13,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:14,266 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:14,688 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:17:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:14,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:16,267 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:16,593 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:16,603 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:18,267 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:18,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:18,634 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:19,257 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:20,268 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:22,021 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:22,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:22,058 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:22,268 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:24,269 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:24,271 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:24,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:24,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,270 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:26,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,609 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:17:27,611 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:17:27,611 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:17:27,611 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:17:28,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:17:28,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:29,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:29,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:29,950 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:30,275 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:31,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:31,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:32,276 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:34,276 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:34,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:34,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:35,265 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:36,277 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:37,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:38,277 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:40,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,020 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,278 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:40,920 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:42,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:42,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:42,279 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:44,279 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:44,691 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:17:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:44,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:44,995 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:46,280 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:46,911 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:47,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:47,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,281 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:52,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:52,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:52,281 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:52,562 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:54,282 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:54,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:54,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:56,283 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:57,007 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:57,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:58,229 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:58,283 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:59,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:00,284 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:00,620 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:00,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:00,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,593 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:05,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:05,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,581 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:10,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,632 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,951 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:14,288 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:14,692 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:18:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:15,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,705 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:16,289 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:18,290 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:18,701 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:18,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:19,580 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:20,290 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:20,733 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:20,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:21,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:18:21,928 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:18:21,928 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:18:21,928 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:18:22,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:18:22,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:23,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:23,638 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:23,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:24,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:25,240 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:25,668 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:25,684 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:26,293 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:28,294 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:28,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:28,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:29,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:30,294 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:30,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:30,750 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:31,230 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:32,295 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:33,651 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:33,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,296 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:35,680 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:35,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:36,296 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:36,891 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:38,297 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:39,139 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:39,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:39,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:40,298 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:41,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,298 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:42,888 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:44,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,299 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:44,694 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:18:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:44,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:46,300 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:47,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,105 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:48,301 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:48,550 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:49,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:50,301 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:52,267 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,302 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:54,302 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:54,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,545 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:56,303 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:57,208 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:57,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:58,304 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:59,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:59,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:59,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:00,304 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:02,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,305 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:02,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:03,553 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:19:03,555 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:19:03,555 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:19:03,556 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:19:04,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:19:04,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:04,403 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:04,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:05,198 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:05,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:06,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:07,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:08,307 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:09,360 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:09,374 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,308 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:11,190 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:12,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,309 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:14,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:14,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:14,309 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:14,696 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:19:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:16,310 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:16,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:17,926 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:17,945 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:18,311 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:20,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:20,874 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:21,313 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:22,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:22,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:23,313 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:24,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:25,314 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:27,315 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:27,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:28,508 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:29,315 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:29,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:29,889 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:29,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,316 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:31,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:33,317 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:34,506 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:34,715 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:34,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:35,317 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:36,743 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:36,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:37,318 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:39,319 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:39,756 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:39,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,161 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:41,319 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:41,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:41,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,320 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:43,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:44,698 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:19:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:44,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:45,172 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:19:45,174 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:19:45,174 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:19:45,175 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:45,175 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:19:45,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:19:45,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:46,614 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:46,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:48,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:48,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:49,322 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:51,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:51,323 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:51,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:51,784 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:53,324 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:55,325 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:55,344 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:55,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:56,804 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:57,325 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:58,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:58,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:59,326 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:00,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:00,486 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:01,327 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:02,792 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:03,328 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:03,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:03,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:05,333 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:05,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:05,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:07,333 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:08,453 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:08,677 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:08,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:09,334 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:10,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:10,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:11,335 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:13,335 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:13,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:13,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:14,428 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:14,699 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:20:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:15,336 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:15,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:15,775 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,336 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:18,765 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:18,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:19,337 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:20,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:20,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:20,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:21,337 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:23,338 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:23,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:23,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,093 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:25,338 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:25,843 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:26,781 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:20:26,782 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:20:26,782 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:20:26,783 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:20:27,339 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:20:27,339 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:28,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:28,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:29,340 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:29,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:30,898 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:30,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:31,077 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:31,340 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:33,341 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:34,518 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:34,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:34,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:35,342 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:36,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:37,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,342 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:39,343 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:39,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:39,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,343 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:41,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,726 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:43,344 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:44,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:44,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:44,701 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:20:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:44,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:45,345 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:46,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:46,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:46,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:47,346 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:48,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:48,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:49,346 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:50,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:50,993 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:51,347 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:52,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:53,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:53,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:53,348 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:55,046 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:55,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:55,348 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:57,349 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:57,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:57,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,382 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:59,349 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:00,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:00,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:01,350 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:03,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,351 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:04,376 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:05,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:05,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:05,351 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:07,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,352 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:08,376 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:21:08,378 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:21:08,378 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:21:08,379 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:21:09,353 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:21:09,353 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:09,379 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:10,433 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:10,455 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:10,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:11,354 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:13,354 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:13,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,459 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:14,704 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:21:14,706 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:14,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:15,356 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:15,481 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:15,492 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:17,357 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:18,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,519 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:19,357 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:20,018 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:20,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:20,551 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:21,358 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:23,358 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:23,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,359 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:25,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,997 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:27,359 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:27,520 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:27,536 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:29,360 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:30,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:30,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:31,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:31,361 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:32,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:32,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:33,361 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:35,281 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,362 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:36,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:37,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,325 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,363 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:39,363 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:40,283 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,308 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:41,364 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:41,666 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:42,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:42,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:43,365 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:44,704 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:21:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:44,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:45,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,366 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:47,367 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:47,648 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:48,933 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:48,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:48,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:49,367 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:49,992 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:21:49,993 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:21:49,994 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:21:49,994 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:21:50,368 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:21:51,368 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:51,961 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:51,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:52,661 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:53,369 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:53,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:54,008 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,369 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:56,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:56,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,370 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:58,071 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:58,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:58,311 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:59,370 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:00,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:00,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:01,371 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:03,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,372 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:04,309 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:05,280 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,373 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:07,374 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:08,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:08,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:09,374 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:09,979 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:10,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:10,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,375 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:13,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,376 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:14,705 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:22:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:14,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:15,376 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:15,398 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:15,408 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:15,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:17,377 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:18,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:18,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:19,378 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:20,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:21,378 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:22,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:22,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:23,379 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:25,380 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:26,464 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:26,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:27,380 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:29,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:29,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:29,381 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:29,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:31,382 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:31,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,623 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:31,637 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:22:31,638 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:22:31,638 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:22:31,639 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:22:32,382 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:22:33,383 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:34,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:34,554 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:35,383 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:36,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,640 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:37,384 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:39,385 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:39,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:39,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,385 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:41,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,631 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,656 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:43,386 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:43,651 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:44,707 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:22:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:45,386 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:45,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:45,699 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:46,661 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:47,387 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:47,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:47,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:49,388 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:50,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:50,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,388 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:51,666 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:52,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:52,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:53,389 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:55,390 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:56,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,101 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,672 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:57,390 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:58,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:58,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:59,391 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:01,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,392 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:01,674 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:03,393 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:04,745 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:04,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:04,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:05,393 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:06,694 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:06,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:06,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:07,394 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:08,846 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:09,395 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:11,395 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:11,716 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:11,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:11,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:13,394 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:13,395 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:23:13,395 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:13,396 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:23:13,396 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:13,396 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:13,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:14,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:14,709 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:23:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:14,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:15,397 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:16,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:17,397 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:18,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:18,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:19,398 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:20,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:20,808 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:21,399 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:21,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:22,826 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:22,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:23,399 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:25,400 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:25,730 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:25,745 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:26,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:27,400 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:27,766 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,401 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:29,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,401 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:32,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:32,957 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:32,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:33,402 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:34,995 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:35,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:35,402 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:37,033 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,403 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:37,785 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:39,404 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:40,533 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:40,562 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:40,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:41,404 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:42,806 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:43,405 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:43,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:43,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:44,710 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:23:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:45,406 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:45,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:45,460 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:47,406 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:47,819 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:48,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:48,490 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:49,407 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:50,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:50,523 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:51,408 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:52,832 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:53,408 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:53,618 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:55,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:55,195 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:23:55,196 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:55,196 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:55,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:23:55,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:55,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:55,662 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:57,844 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:58,705 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:58,720 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:59,410 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:00,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:00,751 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:01,411 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:02,866 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:03,411 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:03,775 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:03,793 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,412 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:05,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,412 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:07,890 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:08,800 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:08,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:09,413 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:10,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:10,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:11,414 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:12,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:13,415 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:13,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:13,831 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:14,711 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:24:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:14,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:15,415 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:15,850 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:15,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,416 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:17,902 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:19,381 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:19,402 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,416 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:21,417 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:22,278 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:23,417 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:23,840 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:24,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:24,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,396 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:28,841 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:29,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:31,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:31,478 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:33,953 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:34,447 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:34,465 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,498 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:37,421 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:39,181 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:39,404 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,422 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:41,423 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:41,436 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:41,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:43,423 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:44,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,529 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:44,713 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:24:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:45,425 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:46,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:46,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,425 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:48,551 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:24:48,552 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:24:48,553 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:24:48,553 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:24:49,426 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:24:49,426 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:49,473 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,485 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,554 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:51,427 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:51,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:51,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:53,428 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:54,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:55,207 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:55,429 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:57,430 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:58,146 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:58,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:58,180 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,430 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:00,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:00,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:00,218 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:01,431 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:03,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,432 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:05,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,220 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:05,432 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:07,433 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:08,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:08,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:09,434 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:10,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:11,434 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:13,095 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,435 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:14,715 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:25:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:15,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,162 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,436 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:16,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:17,437 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:18,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:19,438 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:20,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:20,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:21,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:21,439 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:22,574 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:22,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,440 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:24,603 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:24,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:26,157 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:27,441 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:27,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,442 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:29,554 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:30,398 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:25:30,399 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:25:30,400 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:25:30,400 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:25:30,442 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:25:31,401 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:31,443 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:32,443 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:32,632 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:32,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,242 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:36,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,637 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:37,445 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:39,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:39,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:39,445 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:41,334 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:41,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:41,446 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:41,648 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:43,447 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:44,221 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:44,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:44,716 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:25:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:45,448 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:46,257 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,653 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:47,448 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:48,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:48,304 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,449 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:51,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,093 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,450 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:51,654 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:53,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:53,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:53,450 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:55,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,451 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:57,320 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:57,451 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:57,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:57,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:59,452 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:59,946 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:59,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,452 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:02,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:02,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:03,315 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:03,453 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:04,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:04,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:05,454 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:07,455 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:07,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:07,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:08,316 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:09,456 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:10,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:10,029 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,456 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:13,457 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:13,517 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:13,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,982 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:14,718 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:26:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:15,458 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:15,674 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:26:15,675 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:26:15,675 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:26:15,675 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:26:16,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,301 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,458 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:26:17,459 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:18,342 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,352 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,463 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:19,463 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:19,629 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:20,398 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:20,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:21,464 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:23,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,465 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:24,634 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:25,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,171 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,465 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:27,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,225 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,466 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:29,466 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:29,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:29,974 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:30,009 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:31,468 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:32,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:32,048 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,468 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:34,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:34,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:35,287 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:35,470 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:36,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:36,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:37,471 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:38,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:38,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:39,471 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:40,296 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:41,472 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:41,950 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:41,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,472 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:43,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,719 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:26:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:45,473 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:46,290 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:46,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:46,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:47,474 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:49,474 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:50,538 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:50,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:50,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:51,475 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:51,973 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:52,596 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:52,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,476 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:55,451 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,476 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:56,975 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:26:56,977 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:26:56,977 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:26:56,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:56,977 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:26:57,477 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:26:57,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:57,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:58,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:59,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:59,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:59,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:01,479 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:02,325 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,340 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,770 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:03,479 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:04,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:04,373 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:05,480 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:07,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,481 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:08,434 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:09,465 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,481 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:11,482 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:12,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:12,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:13,442 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:13,483 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:14,417 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,721 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:27:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:14,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:15,483 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:16,452 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:16,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:17,484 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:18,467 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:19,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,485 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:21,311 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,486 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:23,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,399 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,486 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:24,462 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:25,487 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:26,744 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:26,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:26,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:27,488 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:29,466 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:29,488 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:29,527 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:29,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:31,489 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:31,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,567 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:33,489 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:34,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:34,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:35,130 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:35,490 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:36,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:37,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:38,459 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:27:38,460 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:27:38,461 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:27:38,461 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:27:38,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:27:39,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:39,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:39,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:40,492 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:41,125 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:41,493 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:41,555 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,567 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,493 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:44,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:44,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:44,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:27:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:45,494 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:46,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,789 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:47,496 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:49,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,371 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,496 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:51,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,402 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,497 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:52,612 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:53,498 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:54,386 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,399 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:55,498 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:56,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:56,432 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,499 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:58,272 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:59,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,451 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,500 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:59,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:01,473 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:01,483 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:01,501 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:03,501 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:04,270 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:04,996 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:05,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:05,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:05,502 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:07,503 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:07,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,893 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,274 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:09,503 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:09,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:11,504 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:12,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:13,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:13,505 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:14,296 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:14,724 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:28:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:14,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:14,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:15,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,061 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,506 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:17,506 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:18,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:18,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:19,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:19,938 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:19,952 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:28:19,953 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:28:19,954 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:28:19,954 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:28:20,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:28:21,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:22,508 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:22,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:22,958 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:23,508 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:24,951 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:24,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,509 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:27,509 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:27,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:27,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:29,510 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:29,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:30,009 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:30,944 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:31,511 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:32,040 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:32,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,511 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:34,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:34,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:35,512 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:36,591 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:36,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:36,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:37,513 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:38,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,958 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:39,514 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:41,514 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:42,176 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:42,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:42,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:42,583 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:43,515 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:44,725 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:28:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:44,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:45,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,058 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,516 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:47,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:47,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:47,517 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:48,261 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:49,517 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:50,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:51,518 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:52,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:52,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:53,518 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:54,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:55,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,519 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:57,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:57,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:57,520 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:59,521 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:59,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:59,914 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:00,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:01,488 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:29:01,490 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:29:01,490 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:29:01,491 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:29:01,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:29:01,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:02,072 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:02,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:02,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:03,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:04,109 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:04,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:05,523 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:05,803 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:06,857 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:06,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:07,524 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:08,894 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:08,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,525 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:10,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:10,938 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:11,483 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:11,526 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:13,527 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:13,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:13,717 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:14,727 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:29:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:14,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:15,527 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:15,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:15,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,471 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:17,529 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:19,333 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:19,356 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,366 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,529 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:21,530 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:22,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:23,132 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:23,531 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:24,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:24,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:25,531 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:26,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:27,532 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:28,133 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:29,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,199 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,533 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:29,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:31,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,264 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,533 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:33,133 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:33,534 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:34,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:34,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:35,534 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:36,263 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:37,535 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:38,146 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:39,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,300 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,536 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:41,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,536 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:43,143 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:29:43,144 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:29:43,144 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:29:43,145 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:29:43,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:29:43,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:44,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:44,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:44,729 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:29:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:44,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:45,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:46,249 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:46,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:47,539 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:48,279 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:48,290 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:49,540 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:49,773 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:51,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,541 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:53,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,252 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,541 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:55,542 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:55,772 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:56,731 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:56,750 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:56,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,543 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:59,544 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:59,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:59,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:00,960 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:01,544 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:01,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:03,545 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:04,696 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:04,708 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:05,545 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:06,414 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:06,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:07,546 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:09,547 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:09,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:09,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,548 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:11,679 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:12,410 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:13,548 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:14,704 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:30:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:15,549 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:16,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:16,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:17,550 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:18,065 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:19,551 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:19,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:19,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:21,551 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:21,865 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:21,874 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,552 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:24,047 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:24,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:30:24,738 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:30:24,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:30:24,738 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:30:24,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:24,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:25,552 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:30:25,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:26,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:26,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:26,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:27,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:29,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:29,711 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:29,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:29,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:29,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:31,554 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:31,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:31,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,555 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:35,469 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:30:35,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,556 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:35,702 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:37,557 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:38,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,262 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:39,558 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:40,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:40,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:41,358 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:41,558 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:43,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,559 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:44,731 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:30:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:44,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:45,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,471 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,560 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:47,263 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:47,561 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:48,436 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:48,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:52,264 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:53,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:53,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:57,264 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:58,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:58,559 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:59,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:00,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:00,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:01,565 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:02,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:03,565 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:03,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:03,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:05,566 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:05,629 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:05,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:07,567 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:08,182 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:08,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:08,575 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:09,568 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:10,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:10,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,568 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:13,569 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:14,151 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:14,174 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:14,177 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:14,190 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:14,733 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:31:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:14,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:15,570 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:16,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:17,570 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:18,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:31:18,194 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:31:18,195 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:31:18,195 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:31:18,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:31:18,961 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:19,196 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:19,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:20,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:20,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:23,572 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:23,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:23,815 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:24,859 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:25,573 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:25,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:27,573 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:28,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:28,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:29,574 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:29,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:29,957 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:31,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,575 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:33,575 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:34,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:35,576 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:35,851 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:36,109 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:36,127 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:37,577 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:39,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,150 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,577 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:41,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:41,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:41,249 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:41,578 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:43,579 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:44,098 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,735 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:31:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:44,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:45,579 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:46,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:46,144 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:47,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:47,580 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:49,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,580 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:51,581 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:52,769 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:52,790 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:53,582 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:55,583 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:55,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:55,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,583 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:57,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,877 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:58,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:59,575 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:31:59,577 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:31:59,577 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:31:59,577 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:31:59,584 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:31:59,584 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:00,585 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:00,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:00,898 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:01,585 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:02,920 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:03,586 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:03,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:05,586 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:06,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:07,587 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:08,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:08,050 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,552 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:09,588 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:11,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,589 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:13,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,590 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:14,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:14,736 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:32:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:14,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:15,590 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:16,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:16,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:17,591 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:18,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:19,561 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:19,591 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:21,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:21,127 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:21,592 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:23,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,593 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:24,565 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:25,594 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:26,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:26,048 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,594 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:28,067 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:28,079 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,595 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:29,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:29,865 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:31,596 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:31,667 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:31,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:33,597 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:34,545 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:34,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:35,542 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:35,597 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:36,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:36,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:37,598 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:39,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,598 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:41,205 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:41,219 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:32:41,220 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:32:41,221 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:32:41,221 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:32:41,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:32:41,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:41,641 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:41,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:42,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:43,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:44,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,738 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:32:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:44,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:45,600 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:46,665 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:46,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:47,190 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:47,601 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:49,602 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:49,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,602 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:51,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:52,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:53,603 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:54,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:54,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,604 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:56,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:56,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:57,604 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:58,850 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:59,605 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:59,686 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:59,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:01,605 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:01,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:01,736 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:03,606 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:04,502 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:04,602 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:04,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:05,606 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:06,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:06,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:07,607 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:09,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:09,608 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:10,201 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:10,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:10,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:11,609 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:13,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,610 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:14,740 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:33:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:15,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,610 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:17,611 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:18,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:18,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:19,612 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:20,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,505 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:21,612 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:22,847 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:33:22,849 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:33:22,849 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:33:22,850 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:33:23,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:33:23,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:24,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:25,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,614 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:26,169 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:27,614 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:28,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:28,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:29,615 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:29,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:30,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:30,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:31,176 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:31,615 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:33,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,616 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:35,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:35,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:35,617 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:37,172 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:37,618 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:38,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:39,618 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:40,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:40,383 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:41,619 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:42,845 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:43,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,619 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:44,741 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:33:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:44,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:45,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:45,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:46,620 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:48,621 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:48,842 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:48,966 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:48,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:49,008 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:50,622 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:51,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:51,802 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:52,622 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:53,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:53,833 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:54,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:54,623 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:56,624 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:56,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:56,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,624 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:58,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,796 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:59,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:00,625 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:01,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:01,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:02,626 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:03,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:04,503 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:34:04,505 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:34:04,505 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:34:04,506 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:34:04,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:34:04,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:05,508 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:06,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:06,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:06,795 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:10,509 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:11,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:11,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:13,772 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:13,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:14,743 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:34:14,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:15,983 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:16,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:16,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:18,631 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:18,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:18,966 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:21,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:22,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:22,024 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:24,044 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:24,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:26,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:27,580 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:27,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:27,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:29,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:30,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:31,939 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:32,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:32,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:34,636 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:35,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:35,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,222 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:37,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:42,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,263 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:44,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:44,940 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:34:45,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,568 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:48,641 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:50,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,072 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,570 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:54,106 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:54,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:57,571 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:58,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:58,857 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:01,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:01,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:02,645 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:02,903 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:05,394 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:05,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:07,904 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:08,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:12,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:13,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:14,933 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:35:15,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:15,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:16,650 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:18,005 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,024 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,182 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:20,065 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:23,183 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:24,900 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:27,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:27,685 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,183 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:29,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:29,715 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:29,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:32,655 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:32,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:32,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:33,942 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:34,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:34,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,507 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:38,943 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:39,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:39,552 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,899 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:42,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:44,658 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:44,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:44,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:35:45,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:45,796 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:46,660 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:47,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:49,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:50,606 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:50,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:54,964 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:55,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:55,486 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,511 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:59,966 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:00,597 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,619 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,664 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:02,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:04,967 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:05,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:05,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:07,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:07,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:09,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:10,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:14,669 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:14,970 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:36:14,971 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:15,322 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:17,355 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:17,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:19,973 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:20,664 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:20,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:24,974 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:25,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:25,453 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,675 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:29,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:30,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,458 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:33,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:33,252 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:38,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:38,085 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,110 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:41,304 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:42,681 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:43,033 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:43,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:44,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:44,877 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:36:45,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:45,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:47,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:47,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:49,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:49,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:51,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:52,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:52,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,772 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:56,685 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:57,525 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:58,147 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:58,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:58,192 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:01,005 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:01,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:02,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:03,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:03,061 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,859 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,882 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,927 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:07,933 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,945 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,689 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:10,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,869 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,902 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:13,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:14,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:14,876 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:37:15,698 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,716 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,760 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:18,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:20,534 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:20,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:22,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:22,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:24,202 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:24,695 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:25,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:25,528 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:29,203 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:29,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:29,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:30,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,350 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:34,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:35,762 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:35,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:35,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,700 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:40,109 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:40,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,449 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:44,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:44,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:44,929 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:37:45,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,503 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,930 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:48,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:48,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:51,112 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:52,704 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:53,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:53,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:56,112 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:58,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:58,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:59,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:59,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:00,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:00,165 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:01,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:02,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:02,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:04,708 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:05,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:05,027 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:07,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:07,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:07,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:10,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:10,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:12,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:13,493 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:13,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:13,521 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:14,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:14,955 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:38:16,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:16,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,663 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:18,712 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:21,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,303 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,314 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:26,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:26,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:29,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:31,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:31,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:32,717 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:33,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:33,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:34,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:36,122 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:36,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:39,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:40,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:40,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:44,419 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:44,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:44,887 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:38:45,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:45,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:46,722 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:47,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:47,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:50,387 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:51,323 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:51,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:51,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:54,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:54,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,389 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:56,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:56,071 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:59,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:59,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:00,726 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:00,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,924 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,937 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:03,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:03,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,760 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,938 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:08,700 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:08,714 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,939 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:13,590 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:13,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:14,730 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:14,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:14,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:14,975 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:39:15,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,976 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:18,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:18,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:23,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:23,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:28,736 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:28,947 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:28,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:28,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:29,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:31,701 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:31,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:31,960 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:33,731 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,725 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,961 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:38,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:38,771 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:41,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:41,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:42,501 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:42,742 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:43,788 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:43,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:44,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:44,893 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:39:46,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:46,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:47,896 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:48,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:48,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:51,414 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:51,424 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:52,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:53,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:53,458 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,747 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:58,548 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:58,602 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:58,629 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:59,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:01,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:01,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,452 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:06,876 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:06,910 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:09,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:09,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:10,753 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:11,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:11,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:14,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:14,918 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:40:16,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:19,921 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:21,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:21,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:25,029 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:26,591 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,757 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:28,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:28,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:29,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:30,906 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:31,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:31,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:35,907 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:36,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:38,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:38,355 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:40,761 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:41,070 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:41,193 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:41,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:44,885 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:44,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,915 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:40:44,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:47,910 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:47,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:52,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:52,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,766 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:54,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:57,104 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:57,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:57,790 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:59,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:59,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:04,658 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:04,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,770 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:07,665 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:07,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:08,019 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:09,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:09,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,690 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:13,020 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:14,729 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:14,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:14,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,976 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:41:17,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:17,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:18,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:19,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:19,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:20,775 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:23,041 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:23,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:23,075 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,036 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:25,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:25,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:29,038 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:29,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:30,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:30,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,780 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:34,987 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:35,619 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:35,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,654 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,988 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:40,613 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:40,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:44,924 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:41:45,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:45,514 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:45,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:47,532 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:47,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:48,785 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:50,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:50,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:51,778 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:52,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:56,779 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:57,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,575 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:59,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:00,892 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:00,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:00,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,617 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:02,789 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:03,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:03,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:08,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:08,592 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,610 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,619 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:13,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:13,457 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:14,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:14,973 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:42:15,479 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:15,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:16,793 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:18,087 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:18,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:18,324 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,352 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,089 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:23,264 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,298 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:28,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,183 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:29,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:30,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,797 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:33,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,948 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:35,216 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:35,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,649 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:38,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,677 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:41,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:41,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,437 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:44,167 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:44,801 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:44,871 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:42:46,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:46,407 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:49,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:51,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:51,376 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,396 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:54,874 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:56,257 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:56,269 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,287 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,806 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:59,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:59,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:01,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:01,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:04,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:06,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:06,140 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:09,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:10,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:11,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:12,810 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:13,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:13,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:14,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:14,883 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:43:15,885 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:16,504 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:16,526 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:16,542 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:20,886 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:21,276 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:21,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:24,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:24,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,387 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:26,816 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:29,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:29,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:31,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:31,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:31,869 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:33,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:33,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:36,870 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:38,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:38,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:40,821 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:40,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:40,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:42,226 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:43,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:43,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:44,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:44,947 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:43:45,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,867 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:47,948 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:48,815 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:52,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:54,399 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:54,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:54,435 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:54,825 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:57,243 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:57,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:58,138 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:59,281 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:59,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:59,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:02,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:02,183 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:03,922 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:04,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:04,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:07,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:07,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:08,830 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:08,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:09,382 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:09,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,083 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:13,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:14,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:14,998 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:14,999 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:15,191 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:44:17,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:17,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:19,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,834 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:24,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:24,955 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:25,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:27,523 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:27,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:29,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:30,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,056 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:34,077 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:35,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:36,839 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:37,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:37,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:39,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:39,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:42,089 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:42,099 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:44,911 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:44:46,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:46,933 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:46,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:50,844 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:52,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,035 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,786 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:54,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:54,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,786 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:59,050 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:01,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:01,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:02,798 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:03,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:03,818 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,848 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:05,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:05,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:07,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:08,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:08,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,212 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:12,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,806 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:14,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:14,891 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:45:15,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:15,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:17,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:17,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:18,143 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:18,853 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:20,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:22,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:22,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:23,144 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:25,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,171 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:27,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:27,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:28,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:29,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,858 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:33,388 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:35,104 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:35,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:38,389 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:40,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:43,390 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:44,946 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:45:45,079 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:45,098 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:46,862 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:47,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:47,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:48,595 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:50,437 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:50,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:50,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,596 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:55,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:55,162 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,798 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:58,866 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:00,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:00,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:03,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:03,163 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:04,380 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:04,868 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:05,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:05,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:09,380 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:09,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:10,001 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:10,870 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:12,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:12,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:14,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:14,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,863 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,978 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:46:17,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:17,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,761 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,979 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:22,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:22,723 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,874 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:25,316 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:28,218 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:28,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:28,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:30,902 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:30,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:30,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:32,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,769 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:37,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,879 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:41,547 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:42,686 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,698 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:44,979 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:46:45,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:45,606 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:46,980 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:47,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:47,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:50,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:50,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:51,981 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:52,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,883 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:55,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,469 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:57,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:59,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,824 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,897 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:02,930 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,590 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:06,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,886 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:08,485 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:09,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,407 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:11,426 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:11,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:13,486 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:14,365 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:14,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:14,884 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:47:16,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:16,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,472 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,490 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,885 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:20,890 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:21,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:21,725 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:24,370 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:25,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:25,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:29,371 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:29,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:30,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:30,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,866 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:34,893 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:35,274 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:35,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:35,694 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,275 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:40,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,519 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:44,545 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,559 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:44,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:44,898 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:47:45,899 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:46,578 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:46,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:48,898 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:49,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:49,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:51,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:51,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:51,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:56,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:57,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:59,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:59,805 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,866 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:02,902 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:04,672 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:04,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:06,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:06,761 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:07,185 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:08,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:08,806 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,924 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:12,186 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:13,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:13,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:14,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:14,931 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:48:15,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:16,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:16,906 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:17,985 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:18,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:18,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,565 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:22,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:25,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:25,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:29,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:30,910 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:31,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,111 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,927 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:35,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:35,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,928 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:40,308 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:40,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:43,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:43,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:44,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:44,896 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:48:44,914 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:45,387 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:45,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,340 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:49,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:50,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:50,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:54,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:55,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,282 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,319 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:58,919 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:59,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:00,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:01,072 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:01,121 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:01,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:05,879 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:06,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:06,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:10,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:11,201 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:11,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:12,923 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:14,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:14,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:14,968 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:49:15,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:16,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:16,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:20,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:21,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:21,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:25,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:26,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,927 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:28,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:28,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:29,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:31,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:33,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,579 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,342 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:40,098 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:40,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:40,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:40,931 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:42,080 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:43,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:43,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:44,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:44,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:49:45,206 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,225 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,266 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,953 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:50,380 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,393 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,425 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:54,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:54,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:54,935 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:57,350 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:58,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:59,394 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:59,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:59,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:02,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:02,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:03,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:04,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,724 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,853 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:09,939 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:11,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:14,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:14,887 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:50:14,889 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:17,741 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:18,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:18,818 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:19,888 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:20,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,845 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,942 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:24,892 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:25,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:25,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:28,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:28,717 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:29,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:29,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:29,935 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:30,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:30,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,784 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,936 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:35,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:35,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:37,946 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:38,004 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:38,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,078 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:42,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:42,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:44,752 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:44,916 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:50:44,931 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:44,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:45,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:48,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:48,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:50,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:50,083 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:51,447 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:51,949 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:52,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:52,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,449 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:56,490 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:56,510 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:59,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:01,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,985 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:04,376 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:04,386 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:05,954 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:06,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:06,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:07,384 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:09,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:09,431 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:12,385 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:14,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:14,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:14,752 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:14,933 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:51:16,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,457 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:17,817 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:19,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,958 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:21,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:21,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:22,858 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:24,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,313 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:27,858 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:29,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:31,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:31,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:33,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:33,962 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:34,871 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:34,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:37,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:37,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:38,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:39,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:42,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:42,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:43,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:44,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:44,939 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:51:47,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:47,609 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:47,966 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:49,417 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:49,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:54,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,482 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,420 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:59,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:59,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:01,971 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:02,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,530 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,899 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:07,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:12,890 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:12,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:12,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:13,974 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:14,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:14,947 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:52:14,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:15,731 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:15,745 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:20,612 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:20,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,645 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:25,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:25,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:29,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:29,979 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:30,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:30,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,971 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:37,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,972 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:42,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:43,983 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:44,446 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:44,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:44,899 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:52:45,900 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:47,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:50,967 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:50,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:53,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:53,871 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,895 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,987 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:56,529 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:58,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:58,884 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:59,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:00,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:01,915 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:03,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:03,877 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:06,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:08,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:08,920 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:09,990 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:10,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:10,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:12,718 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:13,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:14,949 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:53:15,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:17,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:18,957 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:18,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,834 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:23,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,989 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,994 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:26,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:28,835 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:29,387 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:29,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:29,425 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:32,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:32,201 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:33,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:34,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:34,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,998 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:39,121 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,628 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:42,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:42,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,075 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,629 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:44,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:53:46,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:46,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:51,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:51,935 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:52,002 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:53,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:55,527 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:56,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:56,844 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,891 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:00,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:01,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:01,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,897 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:06,005 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:06,501 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:08,454 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:08,471 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:08,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:10,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:10,515 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,502 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:13,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:13,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:14,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:14,896 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:54:15,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:15,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:18,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:20,009 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:20,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:20,334 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:22,513 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:23,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:25,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:25,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:27,513 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:27,966 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:27,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:30,013 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:33,137 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:34,013 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:35,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:35,027 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:37,839 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:37,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:38,138 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:39,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:39,882 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:42,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:42,837 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:43,139 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:44,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:44,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:44,993 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:54:46,015 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:46,393 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:46,411 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:48,442 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:48,995 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:49,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:53,996 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:54,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,361 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,190 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:59,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:59,756 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:00,020 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:01,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:01,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,893 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:06,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:06,197 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,894 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:11,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:11,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:14,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:14,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:14,944 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:55:14,944 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:16,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:16,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,945 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:21,104 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:21,115 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,460 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:24,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,501 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,946 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:27,303 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:28,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:29,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:29,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:30,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:32,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:32,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:35,910 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:36,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:36,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,839 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,890 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:42,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:43,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:43,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:44,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:44,755 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:44,980 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:55:46,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:46,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:46,981 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:48,764 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:48,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,982 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:53,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:53,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,035 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:56,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:57,446 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:58,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:59,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:59,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:02,946 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:03,801 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:03,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:03,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:08,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:10,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:10,301 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:10,546 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,284 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:13,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,569 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:14,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:14,909 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:15,129 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:56:15,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:19,131 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:21,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:21,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:23,875 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:23,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:24,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:24,266 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:26,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:26,935 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:29,267 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:29,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:30,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:31,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,675 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:34,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:35,721 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:35,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:38,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,450 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:40,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,716 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,081 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:44,107 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,764 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:44,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:44,775 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:44,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:56:44,940 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:56:44,943 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:56:45,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:56:45,947 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:46,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:46,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:46,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:47,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:56:48,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:51,123 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:52,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:52,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:54,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:55,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:56,941 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:57,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:58,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:59,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:59,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:59,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:02,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,170 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,931 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:04,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:04,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:06,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:07,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:07,141 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:08,435 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:09,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:09,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:10,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:12,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:12,202 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:12,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,441 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:14,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:14,231 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:14,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:14,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:14,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:14,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:14,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:57:16,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:17,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:18,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:19,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:19,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:19,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:20,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:21,382 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:21,394 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:22,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:24,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:25,084 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:26,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:26,780 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:26,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:26,806 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:28,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:28,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:28,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:29,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:29,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:29,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:30,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:31,075 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:31,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:31,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:33,653 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:33,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:34,066 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:36,067 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:36,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:36,681 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:36,725 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:38,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:38,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:38,724 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:40,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:42,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:42,689 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:43,518 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,528 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:44,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:44,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:44,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:44,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:44,984 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:57:45,023 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:57:45,024 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:57:45,024 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:57:45,024 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:57:45,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:57:45,547 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:45,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:46,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:48,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:48,340 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:48,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:50,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,638 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:52,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,693 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:54,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:54,333 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:55,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:55,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:56,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:57,426 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:57,441 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:58,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:59,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:59,483 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:59,760 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:59,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:59,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:59,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:00,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:02,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:02,758 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:02,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:04,991 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:05,610 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:05,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:06,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:07,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:07,654 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:08,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:10,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:10,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:12,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:12,733 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:12,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,081 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:14,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:14,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:14,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:14,900 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:58:15,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:15,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:16,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:16,613 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:17,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:17,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:18,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:20,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:20,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:20,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:22,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:22,597 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:22,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:22,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:24,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:26,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:26,090 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,615 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:58:26,616 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:58:26,616 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:58:26,617 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:58:27,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:58:27,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:28,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:28,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:28,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:29,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:29,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:29,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:30,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:30,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,170 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported