diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,24724 +1,5209 @@ -2024-03-27 14:10:33,215 INFO StreamThr :1483490 [internal.py:wandb_internal():86] W&B internal server running at pid: 1483490, started at: 2024-03-27 14:10:33.214417 -2024-03-27 14:10:33,217 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status -2024-03-27 14:10:33,219 INFO WriterThread:1483490 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/run-golaq7b9.wandb -2024-03-27 14:10:33,220 DEBUG SenderThread:1483490 [sender.py:send():382] send: header -2024-03-27 14:10:33,273 DEBUG SenderThread:1483490 [sender.py:send():382] send: run -2024-03-27 14:10:33,572 INFO SenderThread:1483490 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files -2024-03-27 14:10:33,572 INFO SenderThread:1483490 [sender.py:_start_run_threads():1136] run started: golaq7b9 with start time 1711545033.213735 -2024-03-27 14:10:33,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: check_version -2024-03-27 14:10:33,578 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: check_version -2024-03-27 14:10:33,605 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: run_start -2024-03-27 14:10:33,931 DEBUG HandlerThread:1483490 [system_info.py:__init__():27] System info init -2024-03-27 14:10:33,931 DEBUG HandlerThread:1483490 [system_info.py:__init__():42] System info init done -2024-03-27 14:10:33,931 INFO HandlerThread:1483490 [system_monitor.py:start():194] Starting system monitor -2024-03-27 14:10:33,931 INFO SystemMonitor:1483490 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-27 14:10:33,932 INFO HandlerThread:1483490 [system_monitor.py:probe():214] Collecting system info -2024-03-27 14:10:33,932 INFO SystemMonitor:1483490 [interfaces.py:start():190] Started cpu monitoring -2024-03-27 14:10:33,933 INFO SystemMonitor:1483490 [interfaces.py:start():190] Started disk monitoring -2024-03-27 14:10:33,934 INFO SystemMonitor:1483490 [interfaces.py:start():190] Started gpu monitoring -2024-03-27 14:10:33,936 INFO SystemMonitor:1483490 [interfaces.py:start():190] Started memory monitoring -2024-03-27 14:10:33,938 INFO SystemMonitor:1483490 [interfaces.py:start():190] Started network monitoring -2024-03-27 14:10:33,980 DEBUG HandlerThread:1483490 [system_info.py:probe():151] Probing system -2024-03-27 14:10:33,986 DEBUG HandlerThread:1483490 [system_info.py:_probe_git():136] Probing git -2024-03-27 14:10:33,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:33,996 DEBUG HandlerThread:1483490 [system_info.py:_probe_git():144] Probing git done -2024-03-27 14:10:33,996 DEBUG HandlerThread:1483490 [system_info.py:probe():199] Probing system done -2024-03-27 14:10:33,997 DEBUG HandlerThread:1483490 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T13:10:33.980515', 'startedAt': '2024-03-27T13:10:33.210069', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--freeze_encoder', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft-frozen-encoder', 'commit': 'e7946df277d73ac75c34c2017b01c6d39e0275cd'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft-frozen-encoder', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2202.863015625, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 3171.209, 'min': 1500.0, 'max': 2250.0}, {'current': 2364.325, 'min': 1500.0, 'max': 2250.0}, {'current': 1572.649, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.311, 'min': 1500.0, 'max': 2250.0}, {'current': 1886.877, 'min': 1500.0, 'max': 2250.0}, {'current': 1977.114, 'min': 1500.0, 'max': 2250.0}, {'current': 2391.085, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.56, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.921, 'min': 1500.0, 'max': 2250.0}, {'current': 3284.409, 'min': 1500.0, 'max': 2250.0}, {'current': 1662.262, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.285, 'min': 1500.0, 'max': 2250.0}, {'current': 2370.48, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.366, 'min': 1500.0, 'max': 2250.0}, {'current': 3291.736, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.415, 'min': 1500.0, 'max': 2250.0}, {'current': 2283.506, 'min': 1500.0, 'max': 2250.0}, {'current': 1713.973, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.618, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.445, 'min': 1500.0, 'max': 2250.0}, {'current': 3316.361, 'min': 1500.0, 'max': 2250.0}, {'current': 1645.973, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.14, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.739, 'min': 1500.0, 'max': 2250.0}, {'current': 3273.321, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.755, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.103, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.275, 'min': 1500.0, 'max': 2250.0}, {'current': 3194.935, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.989, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.119, 'min': 1500.0, 'max': 2250.0}, {'current': 1697.351, 'min': 1500.0, 'max': 2250.0}, {'current': 3275.205, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.717, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.68, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.771, 'min': 1500.0, 'max': 2250.0}, {'current': 3334.278, 'min': 1500.0, 'max': 2250.0}, {'current': 1635.286, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.293, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.902, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.097, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.841, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.869, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.608, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.143, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.214, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.704, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.981, 'min': 1500.0, 'max': 2250.0}, {'current': 3272.364, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.677, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.916, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.783, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.15, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.079, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.617, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.699, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.917, 'min': 1500.0, 'max': 2250.0}, {'current': 3275.536, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.448, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.093, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.343, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.244, 'min': 1500.0, 'max': 2250.0}, {'current': 1844.721, 'min': 1500.0, 'max': 2250.0}, {'current': 3312.551, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.468, 'min': 1500.0, 'max': 2250.0}, {'current': 2342.129, 'min': 1500.0, 'max': 2250.0}, {'current': 2427.115, 'min': 1500.0, 'max': 2250.0}, {'current': 2390.813, 'min': 1500.0, 'max': 2250.0}, {'current': 2441.533, 'min': 1500.0, 'max': 2250.0}, {'current': 2391.094, 'min': 1500.0, 'max': 2250.0}, {'current': 2459.387, 'min': 1500.0, 'max': 2250.0}, {'current': 2458.036, 'min': 1500.0, 'max': 2250.0}, {'current': 2306.435, 'min': 1500.0, 'max': 2250.0}, {'current': 3298.811, 'min': 1500.0, 'max': 2250.0}, {'current': 2370.203, 'min': 1500.0, 'max': 2250.0}, {'current': 2387.939, 'min': 1500.0, 'max': 2250.0}, {'current': 1744.029, 'min': 1500.0, 'max': 2250.0}, {'current': 2438.926, 'min': 1500.0, 'max': 2250.0}, {'current': 3106.564, 'min': 1500.0, 'max': 2250.0}, {'current': 2415.235, 'min': 1500.0, 'max': 2250.0}, {'current': 3301.717, 'min': 1500.0, 'max': 2250.0}, {'current': 2364.152, 'min': 1500.0, 'max': 2250.0}, {'current': 2352.175, 'min': 1500.0, 'max': 2250.0}, {'current': 1650.287, 'min': 1500.0, 'max': 2250.0}, {'current': 3311.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3305.609, 'min': 1500.0, 'max': 2250.0}, {'current': 1651.466, 'min': 1500.0, 'max': 2250.0}, {'current': 1653.023, 'min': 1500.0, 'max': 2250.0}, {'current': 3276.845, 'min': 1500.0, 'max': 2250.0}, {'current': 1653.413, 'min': 1500.0, 'max': 2250.0}, {'current': 3302.647, 'min': 1500.0, 'max': 2250.0}, {'current': 1653.312, 'min': 1500.0, 'max': 2250.0}, {'current': 3303.875, 'min': 1500.0, 'max': 2250.0}, {'current': 1620.896, 'min': 1500.0, 'max': 2250.0}, {'current': 1652.028, 'min': 1500.0, 'max': 2250.0}, {'current': 1653.199, 'min': 1500.0, 'max': 2250.0}, {'current': 3299.115, 'min': 1500.0, 'max': 2250.0}, {'current': 1651.729, 'min': 1500.0, 'max': 2250.0}, {'current': 1651.899, 'min': 1500.0, 'max': 2250.0}, {'current': 2278.731, 'min': 1500.0, 'max': 2250.0}, {'current': 3291.881, 'min': 1500.0, 'max': 2250.0}, {'current': 2372.068, 'min': 1500.0, 'max': 2250.0}, {'current': 1855.559, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.361, 'min': 1500.0, 'max': 2250.0}, {'current': 1968.15, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.484, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.246, 'min': 1500.0, 'max': 2250.0}, {'current': 2443.499, 'min': 1500.0, 'max': 2250.0}, {'current': 2139.871, 'min': 1500.0, 'max': 2250.0}, {'current': 2161.111, 'min': 1500.0, 'max': 2250.0}, {'current': 1718.071, 'min': 1500.0, 'max': 2250.0}, {'current': 2151.675, 'min': 1500.0, 'max': 2250.0}, {'current': 2978.053, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.523, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.328, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.12, 'min': 1500.0, 'max': 2250.0}, {'current': 2168.709, 'min': 1500.0, 'max': 2250.0}, {'current': 2050.918, 'min': 1500.0, 'max': 2250.0}, {'current': 2050.632, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.328, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.254, 'min': 1500.0, 'max': 2250.0}, {'current': 3291.107, 'min': 1500.0, 'max': 2250.0}, {'current': 1660.234, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.604, 'min': 1500.0, 'max': 2250.0}, {'current': 1657.069, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.425, 'min': 1500.0, 'max': 2250.0}, {'current': 1850.263, 'min': 1500.0, 'max': 2250.0}, {'current': 3277.475, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1516.8665618896484}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} -2024-03-27 14:10:33,997 INFO HandlerThread:1483490 [system_monitor.py:probe():224] Finished collecting system info -2024-03-27 14:10:33,997 INFO HandlerThread:1483490 [system_monitor.py:probe():227] Publishing system info -2024-03-27 14:10:33,998 INFO HandlerThread:1483490 [system_monitor.py:probe():229] Finished publishing system info -2024-03-27 14:10:34,002 DEBUG SenderThread:1483490 [sender.py:send():382] send: files -2024-03-27 14:10:34,002 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now -2024-03-27 14:10:34,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: python_packages -2024-03-27 14:10:34,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: python_packages -2024-03-27 14:10:34,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:10:34,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:34,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:10:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:10:34,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: telemetry -2024-03-27 14:10:34,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: config -2024-03-27 14:10:34,207 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:10:34,207 DEBUG SenderThread:1483490 [sender.py:send():382] send: telemetry -2024-03-27 14:10:34,207 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:10:34,207 WARNING SenderThread:1483490 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-03-27 14:10:34,207 DEBUG SenderThread:1483490 [sender.py:send():382] send: telemetry -2024-03-27 14:10:34,513 INFO wandb-upload_0:1483490 [upload_job.py:push():131] Uploaded file /tmp/tmp17fwro16wandb/4j712kp4-wandb-metadata.json -2024-03-27 14:10:34,573 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:34,574 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/requirements.txt -2024-03-27 14:10:34,574 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-metadata.json -2024-03-27 14:10:36,573 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:36,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:36,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:38,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:38,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:39,213 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:41,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:41,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:43,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:43,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:44,214 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:46,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:46,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:48,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:48,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:48,577 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:49,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:10:49,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:10:49,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:10:49,218 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:50,578 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:51,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:51,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:52,579 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:53,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:53,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:54,365 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:54,579 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:55,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:55,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:56,580 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:58,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:58,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:58,581 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:10:59,586 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:00,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:00,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:00,581 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:02,582 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:02,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:02,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:04,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:04,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:04,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:04,583 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:04,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:04,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:05,207 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:05,583 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/config.yaml -2024-03-27 14:11:07,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:07,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:10,585 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:10,711 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:11,425 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:11:11,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:11,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:12,585 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:14,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:14,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:14,586 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:16,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:16,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:16,587 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:16,658 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:18,587 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:19,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:19,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:19,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:19,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:19,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:20,588 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:21,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:21,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:22,253 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:22,589 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:24,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:24,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:24,590 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:26,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:26,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:26,590 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:28,199 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:28,591 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:29,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:29,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:30,501 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:11:30,503 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:11:30,503 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:11:30,503 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:11:30,504 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 14:11:30,504 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:11:30,504 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:11:30,504 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:11:30,592 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:30,592 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:11:31,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:31,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:32,592 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:33,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:33,938 DEBUG SystemMonitor:1483490 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-27 14:11:33,940 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:11:34,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:34,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:34,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:34,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:34,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:34,593 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:36,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:36,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:36,594 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:38,594 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:39,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:39,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:39,744 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:40,595 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:40,595 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/config.yaml -2024-03-27 14:11:41,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:41,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:42,596 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:44,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:44,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:44,596 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:45,339 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:46,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:46,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:46,597 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:48,598 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:49,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:49,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:49,897 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:11:49,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:49,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:50,598 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:50,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:52,599 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:52,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:52,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:54,599 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:54,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:54,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:56,229 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:56,600 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:57,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:57,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:58,601 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:11:59,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:59,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:00,601 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:01,837 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:02,602 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:02,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:02,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:03,942 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:12:04,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:04,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:04,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:04,603 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:04,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:04,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:06,604 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:07,795 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:07,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:07,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:08,604 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:09,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:09,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:10,605 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:11,785 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:12:11,786 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:12:11,787 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:12:11,787 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:12:12,606 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:12,606 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:12:12,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:12,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:13,403 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:14,607 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:14,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:14,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:16,608 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:17,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:17,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:18,609 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:19,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:19,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:19,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:19,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:19,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:20,501 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:22,610 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:22,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:22,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:24,611 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:24,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:24,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:25,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:26,611 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:28,111 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:12:28,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:28,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:28,612 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:30,613 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:30,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:30,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:31,524 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:32,614 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:32,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:32,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:33,943 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:12:34,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:34,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:34,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:34,614 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:35,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:35,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:36,615 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:37,492 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:37,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:37,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:38,615 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:40,617 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:40,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:40,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:42,618 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:43,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:43,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:43,082 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:44,618 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:46,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:46,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:46,619 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:48,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:48,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:48,620 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:49,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:49,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:49,156 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:50,621 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:50,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:50,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:52,621 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:52,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:52,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:53,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:12:53,032 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:12:53,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:12:53,032 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:12:53,622 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:12:54,622 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:54,667 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:55,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:55,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:56,623 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:57,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:57,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:58,623 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:12:59,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:59,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:00,624 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:00,645 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:02,625 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:02,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:02,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:03,945 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:13:04,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:04,007 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:04,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:04,625 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:06,262 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:06,451 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:13:06,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:06,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:06,626 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:08,627 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:09,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:09,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:10,627 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:11,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:11,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:12,229 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:12,628 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:14,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:14,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:14,628 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:16,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:16,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:16,629 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:17,852 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:18,630 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:18,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:18,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:19,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:19,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:21,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:21,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:22,632 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:23,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:23,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:23,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:24,632 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:26,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:26,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:26,633 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:28,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:28,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:28,634 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:29,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:30,634 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:31,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:31,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:32,635 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:33,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:33,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:33,947 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:13:34,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:34,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:34,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:34,186 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:13:34,187 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:13:34,188 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:13:34,188 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:13:34,635 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:34,636 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:13:35,188 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:36,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:36,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:36,636 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:37,636 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:38,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:38,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:38,637 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:40,637 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:40,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:41,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:41,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:42,638 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:44,639 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:45,202 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:13:45,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:45,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:46,640 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:46,774 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:48,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:48,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:48,641 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:49,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:49,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:49,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:50,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:50,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:52,391 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:52,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:53,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:53,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:54,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:55,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:55,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:56,644 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:13:58,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:58,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:58,357 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:58,645 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:00,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:00,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:00,645 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:02,646 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:03,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:03,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:03,949 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:14:03,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:04,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:04,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:04,647 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:05,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:05,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:06,647 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:08,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:08,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:08,648 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:08,957 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:10,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:10,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:10,649 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:12,649 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:13,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:13,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:14,650 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:14,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:15,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:15,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:15,600 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:14:15,601 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:14:15,602 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:14:15,602 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:14:15,650 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:14:16,651 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:17,651 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:18,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:18,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:18,652 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:19,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:19,008 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:20,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:20,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:20,228 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:20,653 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:22,654 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:23,616 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:14:23,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:23,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:24,655 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:25,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:26,655 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:26,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:26,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:28,656 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:28,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:28,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:30,656 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:31,500 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:31,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:31,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:32,657 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:33,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:33,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:33,950 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:14:34,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:34,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:34,658 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:36,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:36,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:36,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:37,127 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:38,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:38,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:38,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:40,660 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:41,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:41,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:42,661 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:43,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:43,828 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:43,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:44,661 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:45,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:45,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:46,662 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:48,662 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:48,736 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:48,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:48,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:49,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:49,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:49,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:50,663 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:50,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:50,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:52,664 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:52,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:52,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:54,665 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:54,704 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:55,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:55,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:56,665 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:57,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:14:57,021 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:14:57,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:14:57,022 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:14:57,666 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:14:57,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:58,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:58,666 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:14:59,666 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:00,340 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:00,667 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:01,680 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:15:02,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:02,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:02,667 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:03,951 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:15:04,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:04,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:04,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:04,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:04,668 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:06,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:06,669 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:07,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:07,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:08,669 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:09,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:09,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:10,670 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:11,956 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:12,671 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:12,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:12,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:14,672 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:14,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:14,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:16,672 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:17,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:17,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:17,937 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:19,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:19,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:19,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:19,673 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:19,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:19,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:21,674 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:22,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:22,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:23,560 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:23,675 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:24,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:24,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:25,676 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:27,676 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:27,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:27,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:29,517 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:29,677 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:29,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:29,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:31,678 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:32,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:32,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:33,679 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:33,953 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:15:34,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:34,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:34,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:34,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:35,163 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:35,680 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:37,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:37,681 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:37,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:38,483 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:15:38,484 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:15:38,485 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:15:38,485 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:15:38,681 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:15:39,681 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:41,123 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:41,377 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:15:41,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:41,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:41,682 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:43,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:43,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:43,683 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:45,684 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:46,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:46,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:46,746 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:47,685 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:48,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:48,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:49,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:49,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:51,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:51,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:51,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:52,718 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:53,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:53,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:53,687 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:55,688 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:56,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:56,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:57,688 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:15:58,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:58,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:58,376 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:59,689 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:00,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:00,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:01,689 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:03,045 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:03,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:03,690 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:03,955 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:16:03,956 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:04,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:04,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:04,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:05,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:05,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:05,691 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:07,691 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:08,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:08,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:09,331 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:09,692 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:10,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:10,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:11,693 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:13,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:13,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:13,693 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:14,947 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:15,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:15,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:15,694 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:17,695 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:19,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:19,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:19,209 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:16:19,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:19,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:19,695 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:19,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:19,968 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:16:19,970 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:16:19,970 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:16:19,971 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:16:20,696 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:16:21,696 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:22,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:22,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:23,697 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:24,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:24,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:25,697 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:25,938 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:27,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:27,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:27,698 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:29,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:29,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:29,698 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:31,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:31,699 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:32,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:32,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:33,700 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:33,956 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:16:34,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:34,009 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:34,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:34,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:34,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:35,700 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:36,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:36,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:37,547 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:37,701 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:39,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:39,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:39,702 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:41,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:41,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:41,702 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:43,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:43,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:43,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:43,704 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:45,704 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:46,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:46,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:47,705 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:48,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:48,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:48,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:49,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:49,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:49,706 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:51,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:51,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:51,706 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:53,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:53,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:53,707 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:54,182 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:55,707 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:57,151 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:16:57,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:57,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:57,708 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:16:59,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:59,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:59,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:01,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:01,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:03,958 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:17:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:04,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:04,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:04,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:04,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:05,191 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:06,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:06,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:09,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:09,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:10,432 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:11,711 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:11,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:11,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:12,773 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:17:12,774 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:17:12,775 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:17:12,776 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:17:13,712 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:13,712 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:17:14,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:14,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:15,712 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:15,781 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:16,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:16,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:17,713 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:18,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:18,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:19,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:19,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:19,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:19,714 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:21,431 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:21,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:21,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:21,714 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:23,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:23,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:23,715 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:25,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:26,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:26,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:27,413 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:27,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:28,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:28,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:29,717 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:31,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:31,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:31,718 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:33,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:33,718 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:33,959 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:17:34,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:34,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:34,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:35,170 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:17:35,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:35,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:35,719 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:37,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:37,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:37,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:39,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:39,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:40,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:40,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:41,721 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:42,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:42,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:43,722 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:44,670 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:45,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:45,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:45,723 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:47,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:47,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:47,723 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:49,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:49,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:49,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:49,724 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:50,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:50,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:50,649 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:51,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:52,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:52,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:53,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:54,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:17:54,660 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:17:54,660 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:17:54,661 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:17:54,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:17:55,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:55,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:55,662 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:55,726 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:57,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:57,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:57,726 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:17:59,727 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:00,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:00,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:01,132 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:01,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:02,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:02,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:03,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:03,960 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:18:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:04,010 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:04,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:05,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:05,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:05,729 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:07,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:07,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:07,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:07,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:09,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:10,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:10,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:11,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:12,743 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:13,731 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:14,157 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:18:14,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:14,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:15,732 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:17,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:17,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:17,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:18,734 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:19,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:19,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:19,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:19,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:21,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:22,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:22,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:23,735 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:24,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:24,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:24,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:25,735 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:27,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:27,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:27,736 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:29,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:29,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:29,737 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:30,361 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:31,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:32,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:32,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:33,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:33,962 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:18:34,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:34,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:34,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:34,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:35,739 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:35,989 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:36,002 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:18:36,003 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:18:36,004 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:18:36,004 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:18:36,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:36,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:36,739 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:18:37,740 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:39,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:39,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:39,741 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:41,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:41,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:41,741 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:41,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:43,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:44,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:44,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:45,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:46,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:46,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:47,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:47,743 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:49,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:49,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:49,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:49,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:49,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:49,744 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:51,745 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:52,919 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:18:52,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:52,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:53,580 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:53,746 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:55,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:55,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:55,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:57,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:18:57,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:57,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:59,220 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:59,748 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:00,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:00,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:01,749 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:02,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:02,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:03,750 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:03,964 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:19:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:04,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:04,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:05,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:05,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:05,188 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:05,750 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:07,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:07,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:07,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:09,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:09,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:10,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:10,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:11,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:12,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:12,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:13,753 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:14,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:14,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:15,754 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:16,796 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:17,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:19:17,486 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:19:17,487 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:19:17,487 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:19:17,755 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:17,755 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:19:17,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:17,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:19,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:19,755 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:19,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:19,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:21,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:22,439 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:22,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:22,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:23,757 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:24,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:24,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:25,758 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:27,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:27,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:27,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:28,418 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:29,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:31,288 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:19:31,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:31,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:31,760 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:33,761 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:33,966 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:19:33,968 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:34,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:34,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:34,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:34,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:35,761 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:36,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:36,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:37,762 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:39,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:39,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:39,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:39,763 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:41,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:41,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:41,764 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:43,765 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:44,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:44,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:44,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:45,765 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:46,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:46,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:47,766 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:49,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:49,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:49,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:49,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:49,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:49,767 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:50,637 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:51,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:51,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:51,768 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:53,768 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:54,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:54,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:55,652 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:55,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:56,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:56,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:57,770 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:58,980 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:19:58,981 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:19:58,981 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:19:58,982 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:19:59,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:59,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:59,771 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:19:59,771 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:20:01,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:01,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:01,641 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:01,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:03,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:03,966 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:20:04,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:04,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:04,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:04,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:04,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:05,773 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:06,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:06,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:07,288 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:07,774 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:09,775 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:10,136 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:20:10,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:10,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:11,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:13,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:13,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:13,265 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:13,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:15,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:15,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:15,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:17,778 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:17,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:18,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:18,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:19,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:19,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:19,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:19,779 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:20,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:20,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:21,779 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:23,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:23,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:23,780 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:24,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:25,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:25,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:25,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:27,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:28,408 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:28,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:29,782 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:30,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:30,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:30,543 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:31,782 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:33,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:33,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:33,783 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:33,968 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:20:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:34,011 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:34,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:35,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:35,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:35,784 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:36,520 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:37,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:38,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:38,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:39,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:40,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:40,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:40,501 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:20:40,502 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:20:40,503 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:20:40,503 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:20:40,786 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:20:41,786 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:42,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:43,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:43,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:43,787 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:45,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:45,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:45,788 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:47,788 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:48,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:48,934 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:20:48,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:48,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:49,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:49,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:49,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:49,789 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:51,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:51,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:51,790 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:53,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:53,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:53,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:53,790 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:55,791 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:56,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:56,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:57,792 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:20:58,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:58,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:59,724 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:59,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:01,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:01,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:01,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:03,794 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:03,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:03,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:03,970 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:21:04,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:04,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:04,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:05,216 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:05,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:06,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:06,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:07,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:08,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:08,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:09,796 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:10,342 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:11,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:11,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:11,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:13,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:13,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:13,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:15,798 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:16,334 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:16,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:16,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:17,799 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:18,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:18,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:19,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:19,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:19,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:19,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:21,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:21,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:21,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:21,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:21,980 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:21:21,981 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:21:21,981 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:21:21,982 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:21:22,801 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:21:23,801 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:23,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:23,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:25,802 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:27,802 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:27,951 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:28,419 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:21:28,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:28,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:29,803 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:30,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:30,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:31,804 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:33,570 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:33,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:33,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:33,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:33,972 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:21:34,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:34,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:34,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:35,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:35,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:35,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:37,806 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:38,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:38,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:39,550 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:39,806 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:40,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:40,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:41,807 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:43,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:43,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:43,808 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:45,187 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:45,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:45,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:45,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:47,810 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:48,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:48,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:49,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:49,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:49,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:49,810 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:50,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:50,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:51,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:51,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:53,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:53,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:53,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:55,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:55,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:55,813 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:56,788 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:57,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:21:58,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:58,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:59,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:00,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:00,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:01,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:02,768 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:03,444 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:22:03,446 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:22:03,446 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:22:03,446 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:22:03,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:03,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:03,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:03,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:22:03,973 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:22:04,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:04,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:04,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:05,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:07,203 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:22:07,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:07,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:07,817 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:08,395 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:09,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:10,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:10,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:11,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:12,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:12,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:13,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:14,362 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:15,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:15,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:15,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:17,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:17,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:17,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:19,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:19,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:19,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:19,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:20,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:20,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:21,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:22,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:22,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:23,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:25,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:25,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:25,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:25,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:27,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:27,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:27,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:29,824 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:30,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:30,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:31,623 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:31,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:32,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:32,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:33,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:33,975 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:22:34,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:34,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:34,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:35,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:35,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:35,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:37,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:37,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:37,597 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:37,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:39,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:40,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:40,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:41,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:42,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:42,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:43,254 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:43,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:44,940 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:22:44,941 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:22:44,942 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:22:44,942 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:22:45,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:45,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:22:45,934 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:22:45,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:46,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:47,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:48,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:48,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:49,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:49,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:49,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:49,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:49,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:50,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:50,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:51,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:53,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:53,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:53,832 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:54,218 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:55,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:55,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:55,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:57,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:58,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:58,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:59,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:22:59,851 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:00,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:00,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:01,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:03,828 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:03,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:03,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:03,977 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:23:04,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:04,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:04,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:05,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:05,838 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:05,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:05,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:07,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:08,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:08,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:09,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:10,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:10,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:10,842 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:11,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:13,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:13,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:13,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:15,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:15,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:15,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:16,398 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:17,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:18,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:18,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:19,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:19,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:20,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:20,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:22,156 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:24,489 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:23:24,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:24,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:27,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:27,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:27,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:29,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:29,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:29,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:31,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:32,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:32,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:32,635 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:33,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:33,978 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:23:34,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:34,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:34,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:34,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:34,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:35,845 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:37,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:37,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:37,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:38,054 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:38,068 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:23:38,069 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:23:38,069 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:23:38,070 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:23:38,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:23:39,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:39,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:39,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:41,847 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:42,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:42,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:43,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:44,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:44,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:44,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:45,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:47,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:47,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:47,849 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:49,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:49,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:49,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:49,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:49,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:49,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:51,241 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:51,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:52,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:52,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:53,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:54,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:54,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:55,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:56,333 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:57,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:57,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:57,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:23:59,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:59,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:59,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:01,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:02,311 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:03,831 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:24:03,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:03,980 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:24:04,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:04,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:04,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:04,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:04,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:05,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:06,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:06,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:07,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:07,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:08,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:08,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:09,856 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:11,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:11,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:11,857 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:13,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:13,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:13,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:13,857 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:15,858 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:16,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:16,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:17,858 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:18,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:18,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:19,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:19,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:19,493 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:24:19,859 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:20,734 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:24:20,734 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:20,734 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:24:20,735 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:24:20,859 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:24:21,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:21,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:21,859 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:23,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:23,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:23,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:25,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:26,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:26,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:26,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:27,861 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:28,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:28,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:29,861 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:31,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:31,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:31,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:32,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:33,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:33,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:33,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:33,982 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:24:34,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:34,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:34,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:35,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:36,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:36,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:37,709 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:37,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:38,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:38,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:39,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:41,627 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:24:41,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:41,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:41,865 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:43,676 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:43,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:44,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:44,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:45,867 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:46,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:46,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:47,869 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:48,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:48,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:48,702 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:49,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:49,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:49,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:49,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:51,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:51,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:51,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:53,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:53,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:53,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:54,685 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:55,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:55,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:55,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:57,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:24:58,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:58,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:59,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:00,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:00,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:00,339 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:00,352 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:25:00,353 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:25:00,354 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:25:00,354 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:25:00,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:25:01,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:03,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:03,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:03,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:03,984 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:25:04,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:04,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:04,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:05,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:05,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:05,875 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:06,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:07,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:08,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:08,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:09,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:10,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:10,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:11,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:11,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:13,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:13,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:13,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:15,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:15,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:15,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:17,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:17,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:19,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:19,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:19,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:19,500 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:25:19,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:19,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:19,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:21,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:21,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:21,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:23,566 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:23,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:24,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:24,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:25,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:26,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:26,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:27,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:29,534 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:29,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:29,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:29,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:31,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:31,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:32,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:33,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:33,985 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:25:34,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:34,013 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:34,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:34,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:34,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:35,166 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:35,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:37,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:37,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:37,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:39,886 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:39,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:39,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:40,173 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:41,830 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:25:41,832 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:25:41,832 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:25:41,833 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:25:41,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:41,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:25:42,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:42,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:43,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:45,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:45,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:45,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:46,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:47,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:47,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:47,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:49,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:49,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:49,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:49,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:50,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:50,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:51,778 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:51,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:52,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:52,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:53,891 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:55,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:55,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:55,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:57,754 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:57,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:25:58,850 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:25:58,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:58,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:59,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:01,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:01,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:01,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:03,401 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:03,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:03,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:03,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:03,987 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:26:04,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:04,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:04,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:05,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:06,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:06,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:07,896 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:08,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:08,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:09,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:09,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:11,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:11,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:11,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:13,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:13,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:13,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:15,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:15,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:16,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:16,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:17,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:18,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:18,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:19,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:19,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:19,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:19,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:21,000 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:21,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:21,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:21,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:23,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:26:23,326 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:26:23,327 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:26:23,327 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:26:23,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:23,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:23,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:23,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:26:25,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:26,639 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:26,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:26,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:27,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:28,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:28,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:29,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:31,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:31,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:31,904 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:32,622 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:33,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:33,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:33,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:33,989 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:26:34,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:34,014 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:34,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:35,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:37,395 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:26:37,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:37,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:37,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:38,262 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:39,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:40,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:40,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:41,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:42,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:42,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:43,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:44,235 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:45,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:45,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:45,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:47,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:47,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:47,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:49,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:49,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:49,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:49,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:50,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:50,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:50,770 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:51,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:52,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:52,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:53,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:55,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:55,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:55,848 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:55,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:57,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:57,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:57,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:26:59,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:00,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:00,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:01,481 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:01,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:02,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:02,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:03,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:03,990 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:27:04,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:04,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:04,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:04,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:04,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:04,841 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:27:04,842 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:27:04,842 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:27:04,843 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:27:04,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:27:05,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:06,486 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:07,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:07,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:07,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:09,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:09,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:09,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:11,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:12,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:12,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:12,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:13,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:15,835 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:27:15,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:15,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:15,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:17,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:18,108 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:18,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:18,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:19,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:19,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:19,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:19,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:20,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:20,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:21,920 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:23,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:23,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:23,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:24,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:25,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:26,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:26,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:27,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:28,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:29,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:29,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:29,923 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:31,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:31,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:31,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:33,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:33,991 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:27:34,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:34,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:34,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:34,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:34,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:35,714 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:35,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:36,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:36,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:37,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:39,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:39,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:39,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:41,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:41,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:41,324 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:41,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:43,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:43,953 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:43,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:45,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:46,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:46,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:46,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:27:46,323 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:27:46,323 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:27:46,324 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:27:46,324 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:46,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:27:47,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:49,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:49,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:49,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:49,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:49,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:49,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:51,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:51,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:51,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:52,284 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:53,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:54,999 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:27:55,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:55,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:55,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:57,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:27:57,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:58,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:58,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:59,935 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:00,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:00,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:01,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:03,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:03,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:03,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:03,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:03,993 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:28:04,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:04,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:04,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:05,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:05,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:05,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:07,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:08,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:08,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:09,291 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:09,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:10,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:10,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:11,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:12,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:12,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:13,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:15,272 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:15,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:15,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:15,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:17,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:17,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:17,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:19,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:19,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:19,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:19,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:19,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:19,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:20,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:21,944 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:22,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:22,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:23,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:25,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:25,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:25,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:26,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:27,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:27,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:27,605 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:28:27,607 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:28:27,607 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:28:27,608 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:28:27,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:27,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:28:29,947 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:30,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:30,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:31,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:31,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:33,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:34,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:34,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:34,048 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:28:34,049 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:36,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:36,954 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:37,102 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:28:37,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:37,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:38,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:39,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:39,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:40,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:42,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:42,595 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:42,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:42,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:44,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:44,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:44,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:46,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:47,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:47,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:48,585 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:48,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:49,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:49,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:49,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:49,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:49,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:51,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:51,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:52,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:54,239 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:54,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:54,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:54,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:56,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:56,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:56,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:58,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:58,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:28:59,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:00,221 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:00,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:01,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:01,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:02,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:03,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:03,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:03,996 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:29:04,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:04,016 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:04,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:04,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:05,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:06,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:06,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:06,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:08,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:08,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:09,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:29:09,360 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:29:09,360 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:29:09,360 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:29:09,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:09,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:29:10,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:10,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:11,026 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:12,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:12,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:12,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:14,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:16,597 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:29:16,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:16,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:17,004 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:17,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:18,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:19,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:19,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:19,205 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:19,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:20,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:21,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:22,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:22,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:22,656 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:23,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:24,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:24,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:25,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:27,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:27,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:27,968 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:28,645 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:29,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:29,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:29,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:31,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:31,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:31,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:33,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:33,998 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:29:33,998 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:34,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:34,015 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:34,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:34,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:34,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:35,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:36,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:36,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:37,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:38,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:38,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:39,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:41,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:41,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:43,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:43,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:44,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:46,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:46,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:48,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:48,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:49,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:49,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:49,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:50,715 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:51,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:51,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:51,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:53,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:55,536 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:29:55,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:55,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:55,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:56,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:57,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:57,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:57,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:29:59,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:00,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:00,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:01,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:02,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:02,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:02,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:02,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:30:02,807 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:30:02,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:30:02,808 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:30:02,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:30:03,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:03,999 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:30:04,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:04,016 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:04,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:04,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:04,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:04,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:05,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:07,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:07,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:07,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:07,981 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:09,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:09,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:09,982 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:11,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:11,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:11,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:12,778 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:13,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:14,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:14,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:15,984 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:16,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:16,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:17,780 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:17,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:18,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:18,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:19,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:19,016 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:19,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:19,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:20,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:20,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:21,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:22,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:23,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:23,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:23,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:25,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:25,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:25,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:27,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:27,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:27,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:28,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:29,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:31,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:32,322 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:30:32,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:32,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:33,828 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:33,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:34,001 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:30:34,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:34,016 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:34,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:34,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:34,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:35,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:37,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:37,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:37,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:38,845 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:39,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:39,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:39,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:41,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:41,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:41,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:43,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:43,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:44,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:44,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:44,568 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:30:44,572 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:30:44,573 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:30:44,574 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:30:44,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:30:45,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:46,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:46,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:46,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:47,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:49,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:49,018 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:49,237 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:49,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:49,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:51,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:51,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:52,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:54,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:54,312 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:54,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:54,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:56,001 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:56,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:56,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:58,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:30:59,317 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:59,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:59,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:00,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:01,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:01,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:02,003 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:04,002 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:31:04,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:04,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:04,016 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:04,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:04,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:04,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:04,992 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:06,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:06,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:06,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:08,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:10,006 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:10,217 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:31:10,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:10,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:10,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:12,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:12,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:12,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:14,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:14,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:14,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:16,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:16,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:17,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:17,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:18,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:19,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:19,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:19,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:19,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:19,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:20,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:21,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:22,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:22,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:22,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:24,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:24,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:24,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:26,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:26,693 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:27,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:27,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:28,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:28,368 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:31:28,370 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:31:28,370 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:31:28,371 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:31:29,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:31:29,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:29,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:30,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:31,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:31,708 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:31,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:31,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:32,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:34,004 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:31:34,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:34,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:34,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:34,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:34,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:34,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:36,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:36,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:36,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:37,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:38,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:39,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:39,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:40,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:42,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:42,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:42,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:43,357 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:44,019 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:45,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:45,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:46,020 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:48,021 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:48,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:49,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:49,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:49,046 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:31:49,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:49,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:50,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:51,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:51,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:52,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:53,367 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:54,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:54,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:54,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:56,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:56,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:56,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:58,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:31:58,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:58,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:59,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:00,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:00,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:00,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:02,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:03,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:03,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:04,005 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:32:04,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:04,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:04,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:04,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:05,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:05,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:05,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:06,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:08,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:09,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:09,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:10,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:10,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:32:10,024 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:32:10,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:32:10,026 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:32:10,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:10,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:32:11,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:11,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:12,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:13,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:14,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:14,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:14,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:15,027 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:16,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:16,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:16,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:18,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:18,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:18,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:19,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:19,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:20,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:20,742 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:20,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:20,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:22,032 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:22,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:22,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:24,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:25,832 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:26,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:27,743 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:32:27,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:27,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:28,034 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:29,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:29,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:30,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:31,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:32,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:32,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:32,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:34,007 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:32:34,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:34,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:34,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:34,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:34,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:34,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:36,037 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:36,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:37,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:37,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:38,037 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:39,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:39,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:40,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:41,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:41,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:41,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:42,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:44,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:44,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:44,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:46,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:46,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:46,463 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:48,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:49,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:49,017 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:49,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:49,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:50,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:50,655 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:32:50,656 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:32:50,657 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:32:50,657 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:32:51,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:32:51,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:51,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:51,658 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:52,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:53,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:54,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:54,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:54,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:56,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:56,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:56,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:57,314 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:58,043 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:32:58,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:58,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:00,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:01,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:01,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:02,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:02,319 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:04,009 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:33:04,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:04,018 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:04,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:06,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:06,442 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:33:06,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:06,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:08,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:08,307 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:09,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:09,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:10,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:11,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:11,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:12,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:13,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:14,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:14,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:14,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:16,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:16,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:16,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:18,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:18,332 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:18,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:18,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:19,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:19,019 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:20,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:21,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:21,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:22,056 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:23,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:23,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:23,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:24,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:25,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:25,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:26,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:28,059 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:28,609 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:28,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:28,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:30,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:30,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:30,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:32,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:33,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:33:33,612 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:33:33,612 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:33:33,612 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:33,613 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:33:33,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:33,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:34,011 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:33:34,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:34,019 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:34,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:34,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:34,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:33:35,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:35,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:35,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:38,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:38,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:38,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:38,972 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:40,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:40,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:40,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:42,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:44,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:44,634 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:33:44,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:44,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:44,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:46,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:47,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:47,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:48,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:49,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:49,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:49,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:49,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:50,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:50,222 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:52,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:52,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:52,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:54,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:54,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:54,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:55,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:56,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:57,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:57,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:58,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:33:59,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:59,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:00,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:00,631 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:02,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:02,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:02,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:04,013 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:34:04,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:04,019 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:04,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:04,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:04,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:06,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:06,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:06,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:06,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:08,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:09,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:09,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:10,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:11,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:11,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:12,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:12,299 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:14,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:14,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:14,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:16,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:16,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:16,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:17,308 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:34:17,309 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:34:17,309 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:34:17,309 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:17,310 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:34:18,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:18,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:34:19,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:19,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:19,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:19,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:19,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:20,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:22,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:23,305 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:23,657 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:34:23,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:23,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:24,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:26,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:26,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:26,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:28,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:28,322 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:28,698 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:28,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:30,078 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:31,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:31,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:32,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:33,328 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:33,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:33,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:34,014 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:34:34,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:34,019 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:34,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:36,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:36,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:36,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:38,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:38,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:38,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:38,995 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:40,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:41,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:41,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:42,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:43,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:43,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:44,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:44,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:46,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:46,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:46,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:48,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:48,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:49,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:49,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:49,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:49,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:50,090 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:51,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:51,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:52,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:53,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:53,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:54,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:54,090 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:56,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:56,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:56,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:58,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:34:58,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:58,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:59,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:34:59,033 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:34:59,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:34:59,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:59,034 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:34:59,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:35:00,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:01,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:02,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:03,215 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:35:03,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:03,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:04,016 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:35:04,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:04,019 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:04,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:04,247 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:05,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:05,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:06,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:08,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:08,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:08,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:10,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:10,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:10,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:10,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:12,100 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:13,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:13,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:14,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:15,063 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:15,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:15,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:16,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:18,103 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:19,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:19,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:19,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:19,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:19,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:20,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:20,242 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:21,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:21,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:22,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:23,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:23,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:24,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:25,730 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:26,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:26,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:26,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:28,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:29,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:29,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:30,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:30,732 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:31,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:31,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:32,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:34,017 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:35:34,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:34,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:34,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:34,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:34,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:35,744 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:36,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:36,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:36,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:38,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:39,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:39,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:40,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:40,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:40,778 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:35:40,779 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:35:40,779 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:35:40,780 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:35:41,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:35:42,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:42,859 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:35:42,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:42,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:43,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:44,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:45,779 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:46,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:46,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:46,112 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:48,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:48,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:48,112 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:49,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:49,020 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:50,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:50,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:50,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:50,796 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:52,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:52,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:52,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:54,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:54,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:54,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:55,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:56,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:56,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:56,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:58,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:35:59,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:59,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:00,116 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:01,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:01,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:02,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:02,116 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:04,019 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:36:04,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:04,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:04,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:04,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:04,117 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:06,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:06,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:07,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:08,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:08,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:12,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:12,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:12,158 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:14,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:14,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:16,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:17,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:17,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:17,572 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:18,121 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:19,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:19,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:19,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:20,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:20,815 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:36:20,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:20,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:22,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:22,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:22,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:24,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:25,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:25,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:26,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:27,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:27,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:27,841 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:28,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:30,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:30,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:30,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:32,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:32,855 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:32,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:32,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:34,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:34,021 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:36:34,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:34,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:34,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:34,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:36,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:36,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:36:36,225 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:36:36,225 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:36:36,226 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:36:37,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:36:37,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:37,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:37,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:38,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:39,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:39,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:39,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:40,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:42,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:42,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:43,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:43,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:44,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:45,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:45,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:46,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:47,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:48,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:48,120 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:48,133 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:49,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:49,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:50,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:50,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:50,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:52,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:52,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:53,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:53,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:54,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:55,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:55,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:56,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:57,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:58,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:36:58,808 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:36:59,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:59,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:00,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:01,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:01,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:02,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:02,959 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:03,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:04,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:04,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:04,022 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:37:04,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:04,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:06,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:06,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:06,143 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:07,983 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:08,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:08,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:08,145 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:10,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:11,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:11,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:12,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:13,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:13,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:13,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:14,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:15,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:15,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:16,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:18,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:18,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:37:18,049 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:37:18,049 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:37:18,050 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:37:18,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:18,156 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:37:18,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:18,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:19,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:19,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:19,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:19,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:20,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:20,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:20,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:22,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:22,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:22,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:24,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:24,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:25,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:25,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:26,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:27,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:27,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:28,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:29,310 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:29,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:29,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:30,163 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:32,164 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:32,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:32,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:34,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:34,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:34,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:34,164 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:34,249 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:37:35,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:36,669 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:37:36,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:36,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:38,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:38,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:40,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:40,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:41,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:41,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:43,167 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:43,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:43,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:45,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:45,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:46,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:46,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:47,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:48,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:48,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:49,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:49,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:49,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:49,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:50,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:50,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:51,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:51,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:52,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:52,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:53,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:54,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:54,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:55,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:56,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:56,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:56,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:57,172 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:37:59,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:00,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:00,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:01,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:01,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:02,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:02,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:03,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:04,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:04,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:04,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:04,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:04,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:04,202 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:38:05,137 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:38:05,138 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:38:05,138 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:38:05,138 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:38:05,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:05,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:38:06,139 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:07,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:07,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:07,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:09,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:09,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:09,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:11,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:11,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:12,864 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:38:12,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:12,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:13,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:15,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:15,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:15,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:16,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:17,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:17,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:17,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:19,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:19,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:19,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:19,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:21,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:21,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:21,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:21,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:23,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:23,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:23,181 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:25,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:25,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:25,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:26,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:27,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:27,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:27,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:29,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:29,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:29,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:31,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:31,951 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:33,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:33,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:33,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:34,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:34,021 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:34,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:34,234 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:38:35,185 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:35,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:35,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:36,994 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:37,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:37,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:37,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:39,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:40,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:40,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:41,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:42,027 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:42,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:42,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:43,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:45,189 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:45,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:45,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:47,089 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:47,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:38:47,103 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:38:47,104 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:38:47,104 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:38:47,189 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:47,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:38:47,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:47,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:49,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:49,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:49,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:49,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:51,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:51,287 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:38:51,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:51,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:52,131 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:53,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:54,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:54,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:55,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:56,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:56,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:57,187 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:57,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:59,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:38:59,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:59,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:01,194 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:02,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:02,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:02,229 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:03,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:04,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:04,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:04,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:04,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:04,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:04,246 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:39:05,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:06,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:06,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:07,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:07,259 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:08,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:08,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:09,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:11,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:11,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:11,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:12,297 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:13,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:13,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:13,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:15,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:16,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:16,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:17,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:17,340 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:18,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:18,316 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:19,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:19,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:19,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:19,204 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:21,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:21,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:21,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:22,379 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:23,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:23,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:23,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:25,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:26,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:26,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:27,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:27,429 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:29,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:39:29,112 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 14:39:29,113 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:39:29,113 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:39:29,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:29,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 14:39:30,376 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:39:30,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:30,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:31,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:33,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:33,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:33,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:34,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:34,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:34,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:34,146 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:39:35,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:35,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:38,148 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:38,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:38,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:40,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:40,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:42,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:42,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:43,149 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:45,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:45,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:47,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:47,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:48,149 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:49,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:49,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:49,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:50,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:50,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:51,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:39:52,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:52,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:53,606 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:55,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:55,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:57,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:57,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:58,607 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:00,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:00,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:02,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:02,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:03,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:40:04,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:04,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:04,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:04,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:04,251 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:40:05,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:05,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:09,253 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:09,497 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:40:09,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:09,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:12,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:12,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:14,253 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:14,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:14,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:16,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:17,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:40:17,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:19,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:19,022 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:19,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:19,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:19,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:20,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:22,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:22,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:24,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:24,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:25,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:27,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:27,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:29,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:29,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:31,164 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:31,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:40:31,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:33,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:34,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:34,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:34,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:34,250 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:40:35,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:35,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:36,252 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:37,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:37,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:39,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:39,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:41,252 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:41,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:41,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:45,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:45,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:45,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:40:46,545 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:48,813 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:40:48,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:48,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:49,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:49,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:49,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:50,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:50,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:52,229 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:52,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:52,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:55,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:55,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:57,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:57,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:57,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:59,239 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:41:00,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:00,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:02,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:02,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:02,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:04,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:04,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:04,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:04,166 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:41:05,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:06,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:08,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:08,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:08,167 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:11,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:11,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:11,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:41:13,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:13,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:13,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:16,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:16,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:18,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:18,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:18,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:19,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:19,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:19,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:21,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:21,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:23,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:23,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:24,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:25,246 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:41:27,073 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:41:27,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:27,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:29,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:29,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:29,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:31,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:31,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:34,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:34,023 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:34,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:34,169 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:41:35,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:35,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:35,170 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:37,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:37,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:37,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:41:40,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:40,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:40,375 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:42,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:42,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:45,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:45,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:45,376 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:47,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:47,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:49,027 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:49,027 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:49,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:50,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:50,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:51,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:51,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:41:52,316 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:52,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:55,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:55,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:56,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:57,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:57,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:00,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:00,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:01,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:02,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:02,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:04,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:04,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:04,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:04,169 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:42:05,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:42:06,170 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:06,700 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:42:07,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:07,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:09,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:09,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:11,171 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:12,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:12,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:14,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:14,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:16,172 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:16,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:16,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:19,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:19,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:19,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:19,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:19,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:19,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:42:21,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:21,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:21,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:24,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:24,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:26,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:26,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:26,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:28,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:28,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:31,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:42:31,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:31,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:31,773 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:33,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:33,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:34,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:34,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:34,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:34,213 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:42:35,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:36,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:37,214 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:38,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:38,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:40,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:40,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:42,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:43,858 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:42:44,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:44,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:45,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:42:47,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:47,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:47,949 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:49,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:49,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:49,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:49,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:50,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:52,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:52,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:53,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:54,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:54,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:57,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:57,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:57,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:42:59,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:59,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:59,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:01,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:02,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:04,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:04,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:04,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:04,177 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:43:04,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:04,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:05,179 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:06,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:06,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:09,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:09,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:10,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:11,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:43:11,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:11,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:13,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:13,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:15,626 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:16,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:16,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:18,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:18,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:19,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:19,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:19,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:21,505 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:21,988 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:43:22,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:22,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:25,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:25,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:25,283 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:43:26,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:27,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:27,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:30,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:30,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:31,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:32,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:32,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:34,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:34,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:34,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:34,177 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:43:35,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:35,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:37,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:37,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:43:37,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:37,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:40,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:40,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:42,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:42,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:42,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:44,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:44,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:47,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:47,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:47,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:49,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:49,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:49,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:49,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:49,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:51,289 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:43:51,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:51,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:52,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:54,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:54,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:56,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:56,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:57,322 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:00,096 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:44:00,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:00,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:03,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:03,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:03,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:03,293 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:44:04,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:04,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:04,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:04,188 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:44:05,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:05,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:07,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:07,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:08,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:09,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:10,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:12,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:12,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:13,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:15,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:15,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:17,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:17,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:17,296 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:44:18,494 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:19,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:19,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:19,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:20,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:20,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:22,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:22,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:24,057 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:25,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:25,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:27,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:27,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:29,685 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:30,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:30,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:31,302 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:44:32,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:32,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:34,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:34,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:34,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:34,175 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:44:35,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:35,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:35,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:39,036 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:44:39,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:39,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:40,177 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:42,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:42,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:43,309 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:44:44,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:44,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:45,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:47,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:47,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:49,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:49,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:49,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:49,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:49,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:51,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:51,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:51,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:54,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:54,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:56,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:56,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:57,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:57,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:44:59,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:59,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:01,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:01,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:02,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:04,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:04,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:04,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:04,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:04,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:04,179 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:45:06,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:06,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:07,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:09,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:09,318 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:45:09,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:11,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:11,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:12,409 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:14,482 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:14,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:17,410 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:18,299 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:45:18,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:18,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:19,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:19,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:19,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:20,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:20,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:22,643 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:23,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:23,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:23,322 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:45:25,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:25,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:27,644 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:28,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:28,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:30,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:30,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:32,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:32,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:32,644 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:34,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:34,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:34,068 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:34,203 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:45:35,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:35,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:37,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:45:37,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:37,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:37,963 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:39,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:39,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:42,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:42,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:42,963 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:44,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:44,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:47,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:47,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:47,964 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:49,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:49,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:49,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:49,330 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:45:49,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:49,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:51,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:51,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:53,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:54,934 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:45:54,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:54,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:57,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:57,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:58,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:59,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:59,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:01,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:01,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:03,333 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:46:03,609 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:04,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:04,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:04,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:04,181 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:46:04,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:04,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:06,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:06,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:08,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:08,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:09,182 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:11,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:11,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:13,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:13,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:14,183 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:15,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:15,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:17,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:46:18,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:18,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:19,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:19,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:19,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:19,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:20,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:20,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:23,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:23,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:24,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:25,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:25,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:29,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:29,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:29,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:46:30,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:32,800 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:46:32,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:32,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:34,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:34,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:34,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:34,160 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:46:35,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:35,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:36,162 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:37,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:37,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:40,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:40,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:41,162 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:42,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:42,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:43,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:46:44,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:44,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:46,352 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:46,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:46,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:49,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:49,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:49,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:50,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:50,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:52,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:52,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:52,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:54,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:54,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:55,349 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:46:57,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:57,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:57,577 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:59,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:59,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:01,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:01,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:02,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:03,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:03,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:04,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:04,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:04,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:04,172 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:47:05,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:05,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:07,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:09,350 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:47:09,352 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:47:09,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:09,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:12,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:12,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:12,633 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:14,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:14,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,635 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:19,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:19,025 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:19,069 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:19,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:19,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:21,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:47:22,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:22,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:22,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:24,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:24,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:27,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:27,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:27,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:29,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:29,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:32,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:32,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:32,711 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:34,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:34,026 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:34,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:34,215 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:47:34,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:34,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:35,363 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:47:36,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:36,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:38,216 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:39,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:39,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:41,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:41,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:43,217 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:44,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:44,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:47,841 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:47:47,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:47,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:48,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:49,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:49,026 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:49,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:49,370 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:47:50,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:50,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:52,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:52,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:54,160 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:55,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:56,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:58,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:58,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:59,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:01,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:01,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:01,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:48:03,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:03,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:04,026 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:04,026 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:04,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:04,224 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:48:04,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:05,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:05,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:08,135 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:08,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:09,225 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:10,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:10,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:12,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:12,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:14,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:14,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:14,541 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:15,378 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:48:17,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:17,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:19,026 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:19,026 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:19,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:19,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:19,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:20,214 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:21,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:21,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:25,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:25,240 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:48:25,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:25,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:27,381 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:48:27,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:27,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:29,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:29,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:30,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:32,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:32,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:34,026 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:34,026 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:34,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:34,166 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:48:34,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:34,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:36,167 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:37,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:37,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:40,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:40,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:41,385 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:48:42,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:43,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:43,135 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:45,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:45,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:47,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:47,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:47,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:49,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:49,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:49,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:50,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:50,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:52,202 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:52,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:52,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:54,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:54,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:55,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:48:57,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:57,218 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:57,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:59,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:59,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:02,218 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:02,880 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:49:02,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:02,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:04,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:04,029 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:04,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:04,255 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:49:05,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:05,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:07,393 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:49:07,464 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:07,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:07,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:09,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:09,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:12,465 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:12,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:12,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:14,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:14,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:16,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:17,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:17,465 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:19,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:19,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:19,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:20,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:20,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:21,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:49:22,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:22,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:22,571 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:25,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:25,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:27,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:27,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:27,572 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:30,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:30,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,573 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:34,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:34,029 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:34,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:34,220 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:49:34,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:35,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:35,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:49:37,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:37,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:38,221 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:41,081 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:49:41,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:41,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:43,222 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:44,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:44,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:46,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:46,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:47,404 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:49:48,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:48,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:49,027 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:49,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:49,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:49,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:51,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:51,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:53,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:53,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:54,184 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:55,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:55,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:57,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:57,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:59,184 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:59,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:59,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:01,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:01,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:01,408 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:50:03,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:03,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:04,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:04,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:04,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:04,195 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:50:04,196 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:05,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:05,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:08,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:08,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:09,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:10,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:10,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:12,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:12,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:14,509 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:15,411 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:50:16,177 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:50:16,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:16,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:19,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:19,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:19,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:19,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:19,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:20,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:21,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:21,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:23,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:23,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:25,177 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:26,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:26,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:28,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:28,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:28,415 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:50:30,720 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:31,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:31,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:33,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:33,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:34,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:34,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:34,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:34,251 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:50:36,252 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:36,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:36,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:38,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:38,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:41,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:41,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:41,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:42,419 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:50:43,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:43,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:48,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:48,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:49,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:49,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:49,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:51,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:51,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:52,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:55,368 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:50:55,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:55,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:56,425 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:50:57,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:58,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:58,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:00,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:00,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:02,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:03,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:03,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:04,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:04,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:04,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:04,228 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:51:05,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:05,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:07,434 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:07,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:07,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:08,428 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:51:10,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:10,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:12,434 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:12,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:12,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:16,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:16,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:17,435 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:18,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:18,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:19,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:19,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:19,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:20,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:20,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:22,433 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:51:22,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:23,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:23,270 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:25,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:25,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:28,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:28,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:28,271 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:30,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:30,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:33,936 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:51:33,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:33,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:34,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:34,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:34,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:34,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:34,198 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:51:34,437 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:51:36,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:36,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:39,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:39,045 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:39,200 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:41,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:41,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:43,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:43,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:44,201 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:45,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:46,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:48,441 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:51:49,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:49,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:49,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:49,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:49,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:49,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:51,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:51,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:53,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:53,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:54,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:56,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:56,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:58,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:58,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:59,225 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:00,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:00,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:02,445 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:52:03,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:03,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:04,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:04,030 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:04,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:04,256 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:52:04,257 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:05,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:05,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:08,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:08,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:09,258 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:12,008 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:52:12,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:12,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:14,259 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:15,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:15,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:16,449 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:52:17,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:17,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:19,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:19,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:19,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:20,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:20,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:21,000 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:22,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:22,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:24,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:24,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:26,001 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:27,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:27,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:29,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:29,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:30,454 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:52:31,425 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:32,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:32,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:34,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:34,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:34,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:34,218 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:52:34,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:34,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:36,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:36,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:37,219 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:39,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:39,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:41,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:41,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:42,703 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:43,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:43,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:44,458 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:52:46,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:46,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:47,704 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:49,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:49,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:49,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:50,603 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:52:50,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:50,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:52,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:52,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:53,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:55,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:55,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:56,462 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:52:57,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:57,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:59,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:00,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:00,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:02,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:02,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:04,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:04,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:04,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:04,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:04,257 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:53:04,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:04,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:07,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:07,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:09,257 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:09,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:09,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:10,466 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:53:11,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:11,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:14,258 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:14,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:14,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:16,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:16,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:18,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:18,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:19,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:19,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:19,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:20,164 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:21,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:21,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:23,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:23,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:24,470 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:53:25,415 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:27,458 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:53:27,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:27,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:30,416 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:30,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:30,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:32,472 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:53:32,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:32,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:34,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:34,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:34,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:34,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:53:35,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:35,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:36,208 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:37,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:37,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:38,474 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:53:40,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:40,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:41,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:42,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:42,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:45,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:45,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:46,477 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:53:47,249 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:47,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:47,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:49,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:49,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:49,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:50,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:50,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:52,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:52,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:53,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:55,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:55,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:57,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:57,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:58,199 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:00,481 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:54:00,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:00,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:02,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:02,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:03,526 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:04,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:04,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:04,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:04,217 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:54:06,368 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:54:06,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:06,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:09,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:09,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:09,219 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:11,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:11,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:14,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:14,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:14,487 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:54:14,570 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:16,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:16,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:19,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:19,031 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:19,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:19,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:19,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:20,951 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:21,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:21,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:24,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:24,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:26,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:26,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:26,491 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:54:26,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:29,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:29,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:31,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:31,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:31,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:33,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:33,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:34,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:34,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:34,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:34,255 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:54:36,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:36,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:37,260 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:38,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:38,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:40,496 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:54:41,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:41,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:42,385 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:44,368 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:54:44,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:44,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:47,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:47,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:47,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:49,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:49,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:49,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:49,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:49,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:51,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:51,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:52,696 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:54,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:54,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:54,501 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:54:56,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:56,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:57,697 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:58,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:58,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:00,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:00,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:02,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:03,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:03,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:04,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:04,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:04,264 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:55:05,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:05,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:06,504 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:55:07,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:07,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:08,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:10,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:10,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:12,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:12,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:13,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:14,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:15,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:17,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:17,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:18,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:19,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:19,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:20,508 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:55:21,259 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:55:21,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:21,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:23,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:23,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:23,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:26,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:26,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:28,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:28,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:28,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:30,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:31,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:33,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:33,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:34,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:34,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:34,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:34,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:34,174 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:55:34,512 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:55:35,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:35,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:38,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:38,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:39,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:40,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:40,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:42,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:42,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:44,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:45,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:45,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:46,516 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:55:47,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:47,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:49,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:49,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:49,184 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:49,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:49,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:52,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:52,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:54,185 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:54,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:54,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:58,146 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:55:58,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:58,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:59,303 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:00,520 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:56:01,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:01,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:03,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:03,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:04,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:04,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:04,197 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:56:05,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:05,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:05,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:08,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:08,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:10,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:10,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:10,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:12,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:12,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:14,524 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:56:15,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:15,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:15,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:17,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:17,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:19,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:19,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:20,854 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:20,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:20,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:22,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:22,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:24,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:24,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:26,527 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:56:26,788 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:26,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:27,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:29,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:29,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:31,789 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:31,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:31,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:34,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:34,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:34,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:34,213 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:56:35,711 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:56:35,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:35,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:37,215 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:38,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:38,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:40,531 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:56:40,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:40,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:42,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:42,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:43,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:45,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:45,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:47,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:47,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:48,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:49,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:49,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:50,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:50,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:52,534 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:56:52,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:52,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:53,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:55,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:55,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:57,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:57,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:58,322 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:00,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:00,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:02,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:02,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:03,323 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:04,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:04,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:04,175 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:57:04,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:05,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:06,538 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:57:07,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:07,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:08,487 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:09,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:09,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:13,489 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:13,569 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:57:13,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:13,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:16,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:16,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:18,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:18,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:18,725 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:19,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:19,032 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:19,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:20,543 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:57:21,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:21,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:23,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:23,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:24,249 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:25,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:25,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:27,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:27,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:29,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:30,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:30,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:32,549 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:57:32,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:32,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:34,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:34,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:34,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:34,248 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:57:35,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:35,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:35,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:37,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:37,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:40,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:40,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:40,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:42,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:42,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:44,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:44,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:46,217 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:46,553 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:57:47,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:47,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:49,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:49,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:49,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:51,283 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:57:51,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:51,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:51,558 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:53,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:53,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:55,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:55,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:56,558 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:58,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:58,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:58,558 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:58:00,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:00,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:02,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:03,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:03,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:04,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:04,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:04,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:04,184 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:58:05,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:05,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:08,186 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:08,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:08,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:10,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:10,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:12,562 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:58:12,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:12,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:13,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:15,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:16,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:18,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:18,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:18,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:19,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:19,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:19,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:21,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:21,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:23,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:23,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:23,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:25,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:25,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:26,566 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:58:28,938 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:29,960 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:58:30,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:30,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:32,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:32,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:33,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:34,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:34,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:34,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:34,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:34,169 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:58:37,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:37,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:38,570 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:58:39,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:39,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:39,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:41,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:41,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:44,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:44,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:44,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:46,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:46,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:48,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:49,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:49,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:49,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:49,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:51,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:51,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:52,574 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:58:53,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:53,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:54,317 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:56,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:56,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:58,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:58,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:59,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:01,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:01,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:03,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:03,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:04,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:04,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:04,194 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:59:04,577 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:59:05,196 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:07,484 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:59:07,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:07,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:10,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:10,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:10,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:12,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:12,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:14,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:14,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:15,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:17,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:17,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:18,581 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:59:19,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:19,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:19,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:19,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:19,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:20,706 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:21,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:21,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:24,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:24,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:25,706 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:26,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:26,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:28,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:28,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:31,003 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:31,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:31,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:32,586 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:59:33,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:33,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:34,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:34,076 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:34,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 14:59:35,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:35,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:36,208 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:38,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:38,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:40,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:40,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:41,208 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:44,363 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:59:44,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:44,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:44,589 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 14:59:46,256 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:46,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:46,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:49,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:49,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:49,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:49,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:51,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:51,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:52,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:53,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:53,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:56,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:56,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:57,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:58,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:58,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:58,593 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:00:01,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:01,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:02,450 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:03,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:03,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:04,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:04,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:04,198 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:00:05,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:05,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:08,199 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:08,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:08,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:10,597 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:00:10,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:10,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:12,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:12,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:13,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:15,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:15,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:17,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:17,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:18,579 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:19,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:19,034 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:19,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:21,412 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:00:21,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:21,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:23,632 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:24,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:24,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:24,601 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:00:26,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:26,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:28,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:28,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:29,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:31,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:31,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:33,635 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:34,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:34,033 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:34,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:34,181 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:00:34,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:34,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:36,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:36,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:38,605 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:00:38,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:39,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:39,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:41,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:41,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:43,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:44,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:44,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:46,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:46,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:48,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:49,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:49,034 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:49,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:49,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:49,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:52,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:52,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:52,613 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:00:54,074 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:55,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:55,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:57,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:57,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:59,074 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:00,855 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:01:00,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:00,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:03,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:03,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:04,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:04,034 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:04,076 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:04,192 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:01:04,193 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:04,624 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:05,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:05,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:08,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:08,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:09,193 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:10,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:10,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:11,781 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:01:11,782 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 15:01:11,783 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 15:01:11,783 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 15:01:11,783 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 15:01:11,784 DEBUG SenderThread:1483490 [sender.py:send():382] send: metric -2024-03-27 15:01:11,784 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:01:11,784 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:01:11,784 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:01:12,627 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:12,627 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:01:13,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:13,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:14,628 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:14,785 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:15,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:15,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:19,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:19,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:19,786 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:19,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:19,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:19,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:20,630 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:21,135 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:21,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:22,631 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:23,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:23,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:24,995 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:25,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:25,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:28,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:28,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:30,411 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:30,633 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:30,633 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/config.yaml -2024-03-27 15:01:31,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:31,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:32,634 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:33,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:33,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:34,101 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:01:34,635 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:34,796 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:34,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:34,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:35,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:35,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:35,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:36,635 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:38,636 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:39,606 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:01:39,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:39,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:40,636 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:41,183 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:42,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:42,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:42,637 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:44,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:44,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:44,638 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:46,234 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:46,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:46,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:46,638 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:48,639 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:49,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:49,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:49,796 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:49,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:49,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:50,640 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:51,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:51,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:51,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:52,640 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:53,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:53,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:54,641 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:56,642 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:56,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:56,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:57,000 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:58,642 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:01:58,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:58,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:00,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:01,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:01,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:02,058 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:02,644 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:04,103 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:02:04,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:04,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:04,645 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:04,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:04,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:04,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:06,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:06,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:06,646 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:07,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:08,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:08,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:08,646 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:10,647 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:11,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:11,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:11,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:02:11,169 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:02:11,169 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:02:11,170 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:02:11,647 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:02:12,171 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:12,648 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:13,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:13,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:14,648 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:16,649 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:16,869 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:02:16,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:16,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:17,209 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:18,650 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:18,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:18,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:19,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:19,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:19,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:20,651 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:20,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:21,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:22,274 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:22,652 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:23,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:23,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:24,652 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:25,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:26,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:26,653 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:27,356 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:28,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:28,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:28,654 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:30,655 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:31,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:31,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:32,412 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:32,655 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:33,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:33,120 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:34,104 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:02:34,656 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:34,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:34,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:34,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:35,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:35,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:36,657 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:37,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:37,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:37,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:38,658 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:40,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:40,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:40,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:42,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:42,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:42,512 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:42,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:44,660 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:45,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:45,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:46,660 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:47,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:47,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:47,569 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:48,661 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:49,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:49,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:49,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:49,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:49,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:50,662 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:52,650 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:52,662 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:53,062 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:02:53,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:53,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:53,351 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:02:53,352 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:02:53,353 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:02:53,353 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:02:53,663 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:02:54,663 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:56,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:56,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:56,664 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:02:57,720 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:58,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:58,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:58,664 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:00,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:00,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:00,665 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:02,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:02,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:02,666 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:02,772 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:04,106 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:03:04,666 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:04,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:04,797 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:04,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:05,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:05,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:06,667 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:07,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:07,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:07,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:08,668 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:09,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:09,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:10,668 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:12,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:12,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:12,669 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:12,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:14,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:14,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:14,670 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:16,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:16,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:16,671 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:17,944 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:18,671 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:19,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:19,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:19,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:19,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:19,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:20,672 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:21,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:21,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:22,673 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:23,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:23,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:23,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:24,673 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:26,674 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:27,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:27,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:28,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:28,675 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:30,675 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:30,824 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:03:30,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:30,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:32,676 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:33,134 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:33,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:34,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:34,107 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:03:34,677 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:34,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:34,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:34,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:35,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:03:35,326 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:03:35,327 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:03:35,328 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:03:35,677 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:03:36,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:36,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:36,678 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:38,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:38,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:38,469 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:38,678 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:40,679 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:41,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:41,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:42,680 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:43,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:43,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:43,544 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:44,680 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:45,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:45,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:46,681 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:48,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:48,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:48,604 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:48,682 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:49,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:49,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:49,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:50,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:50,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:50,682 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:52,683 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:52,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:52,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:53,687 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:54,683 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:55,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:55,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:56,684 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:57,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:57,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:58,685 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:03:58,767 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:59,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:59,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:00,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:02,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:02,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:02,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:03,839 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:04,109 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:04:04,687 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:04,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:04,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:04,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:04,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:04,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:06,691 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:08,605 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:04:08,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:08,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:08,692 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:08,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:10,693 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:11,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:11,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:12,693 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:13,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:13,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:13,964 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:14,694 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:15,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:15,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:16,695 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:17,375 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:04:17,376 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:04:17,377 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:04:17,377 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:04:17,695 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:04:18,696 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:18,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:18,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:19,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:19,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:19,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:19,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:20,696 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:20,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:20,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:22,697 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:22,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:22,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:24,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:24,698 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:25,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:25,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:26,699 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:27,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:27,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:28,699 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:29,175 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:30,700 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:30,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:30,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:32,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:32,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:33,704 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:34,111 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:04:34,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:34,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:34,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:35,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:35,704 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:35,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:35,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:37,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:37,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:40,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:40,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:40,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:43,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:43,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:45,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:46,800 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:04:46,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:46,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:48,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:48,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:49,712 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:49,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:49,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:49,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:50,985 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:51,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:51,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:51,713 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:53,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:53,714 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:53,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:55,715 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:55,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:56,259 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:56,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:57,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:04:58,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:58,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:59,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:00,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:00,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:01,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:01,717 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:03,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:03,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:03,717 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:04,113 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:05:04,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:04,798 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:04,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:05,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:05,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:05,718 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:06,372 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:07,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:07,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:07,719 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:09,719 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:10,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:10,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:11,421 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:11,435 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:05:11,436 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:05:11,436 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:05:11,437 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:05:11,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:11,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:05:12,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:12,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:12,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:13,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:14,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:14,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:15,721 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:16,503 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:17,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:17,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:17,722 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:19,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:19,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:19,722 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:19,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:19,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:19,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:21,575 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:21,723 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:23,458 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:05:23,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:23,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:23,723 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:25,724 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:26,638 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:26,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:26,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:27,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:28,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:28,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:29,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:30,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:30,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:31,687 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:31,726 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:33,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:33,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:33,726 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:34,115 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:05:34,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:34,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:34,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:35,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:35,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:35,727 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:36,742 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:37,727 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:37,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:37,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:39,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:40,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:40,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:41,729 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:41,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:42,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:42,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:43,729 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:44,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:44,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:45,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:46,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:47,731 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:47,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:47,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:49,731 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:49,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:49,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:49,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:50,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:50,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:51,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:52,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:52,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:53,664 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:05:53,665 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:05:53,666 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:05:53,666 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:05:53,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:53,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:05:54,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:55,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:55,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:55,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:57,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:57,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:57,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:57,735 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:05:59,736 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:01,031 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:06:01,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:01,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:01,736 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:02,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:03,737 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:04,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:04,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:04,116 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:06:04,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:04,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:04,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:05,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:06,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:06,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:07,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:07,740 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:08,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:08,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:09,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:11,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:11,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:11,744 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:12,259 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:13,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:13,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:13,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:15,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:15,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:15,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:17,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:17,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:18,316 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:18,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:19,748 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:19,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:19,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:19,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:20,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:20,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:21,749 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:22,398 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:22,412 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:22,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:23,750 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:25,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:25,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:25,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:27,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:27,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:27,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:27,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:29,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:30,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:30,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:31,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:32,549 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:32,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:32,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:33,753 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:34,117 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:06:34,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:34,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:34,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:34,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:34,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:35,753 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:35,947 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:06:35,948 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:06:35,949 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:06:35,949 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:06:36,754 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:36,754 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:06:37,754 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:38,453 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:39,146 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:06:39,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:39,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:39,755 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:41,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:41,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:41,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:43,521 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:43,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:43,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:43,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:45,757 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:46,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:46,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:47,758 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:48,585 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:48,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:48,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:49,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:49,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:49,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:49,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:50,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:50,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:51,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:53,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:53,760 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:54,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:54,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:55,761 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:56,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:56,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:57,762 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:06:58,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:58,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:58,682 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:59,763 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:01,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:01,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:01,764 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:03,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:03,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:03,738 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:03,764 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:04,119 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:07:04,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:04,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:04,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:05,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:05,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:05,765 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:07,766 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:08,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:08,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:08,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:09,766 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:10,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:10,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:11,767 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:12,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:12,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:13,768 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:13,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:15,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:16,139 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:07:17,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:17,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:17,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:17,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:07:17,937 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:07:17,938 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:07:17,938 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:07:18,770 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:18,770 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:07:18,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:19,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:19,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:19,770 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:19,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:19,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:19,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:21,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:21,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:21,771 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:23,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:23,980 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:24,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:24,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:25,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:26,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:26,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:27,773 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:28,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:28,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:29,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:29,774 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:31,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:31,774 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:31,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:33,775 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:33,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:33,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:34,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:34,120 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:07:34,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:34,799 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:34,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:35,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:35,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:35,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:37,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:38,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:38,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:39,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:39,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:40,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:40,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:41,778 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:43,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:43,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:43,778 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:44,714 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:45,779 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:46,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:46,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:47,780 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:48,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:48,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:49,768 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:49,780 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:49,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:49,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:49,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:50,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:50,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:51,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:53,782 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:53,909 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:07:53,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:53,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:54,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:55,783 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:57,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:57,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:57,784 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:59,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:59,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:59,784 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:07:59,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:59,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:07:59,910 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:07:59,911 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:07:59,912 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:08:00,784 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:00,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:08:01,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:02,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:02,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:03,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:04,122 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:08:04,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:04,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:04,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:04,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:04,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:04,960 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:05,786 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:07,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:07,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:07,787 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:09,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:09,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:09,787 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:10,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:11,788 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:12,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:12,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:13,789 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:14,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:14,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:15,132 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:15,790 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:16,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:16,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:17,791 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:19,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:19,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:19,791 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:19,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:19,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:19,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:20,201 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:21,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:21,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:21,792 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:23,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:23,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:23,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:25,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:25,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:26,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:26,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:27,794 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:28,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:28,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:29,794 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:30,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:31,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:32,323 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:08:32,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:32,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:33,796 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:34,123 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:08:34,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:34,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:34,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:35,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:35,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:35,427 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:35,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:37,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:37,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:37,798 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:39,798 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:40,477 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:40,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:40,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:41,799 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:42,192 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:08:42,193 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:08:42,193 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:08:42,194 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:08:42,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:42,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:42,799 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:42,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:08:43,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:44,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:44,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:45,570 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:45,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:47,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:47,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:47,804 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:49,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:49,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:49,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:49,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:49,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:49,803 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:50,643 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:51,804 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:52,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:52,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:53,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:54,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:54,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:55,704 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:55,806 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:57,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:57,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:57,807 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:08:59,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:59,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:59,807 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:00,775 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:01,808 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:02,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:02,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:03,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:04,125 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:09:04,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:04,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:04,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:04,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:04,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:05,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:05,840 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:07,810 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:07,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:08,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:09,811 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:10,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:11,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:11,932 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:09:11,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:11,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:13,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:13,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:13,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:15,813 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:15,981 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:17,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:17,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:17,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:19,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:19,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:19,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:19,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:19,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:19,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:21,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:21,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:21,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:21,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:23,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:24,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:24,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:24,588 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:09:24,590 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:09:24,590 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:09:24,591 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:09:24,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:09:25,817 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:26,320 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:26,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:26,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:26,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:27,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:28,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:28,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:29,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:31,541 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:31,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:31,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:31,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:33,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:33,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:33,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:34,127 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:09:34,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:34,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:34,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:35,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:35,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:35,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:36,729 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:37,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:38,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:38,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:39,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:40,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:40,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:41,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:41,961 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:42,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:42,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:43,824 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:45,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:45,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:45,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:47,207 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:47,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:49,554 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:09:49,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:49,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:49,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:49,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:49,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:49,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:51,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:51,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:52,300 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:53,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:54,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:54,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:55,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:56,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:56,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:57,718 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:57,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:09:58,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:58,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:59,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:01,832 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:01,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:01,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:02,962 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:03,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:03,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:04,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:04,129 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:10:04,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:04,800 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:04,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:05,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:06,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:06,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:07,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:08,173 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:08,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:10:08,189 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:10:08,190 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:10:08,190 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:10:08,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:08,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:10:08,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:08,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:09,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:10,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:10,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:11,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:12,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:13,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:13,457 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:13,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:15,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:16,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:16,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:17,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:18,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:18,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:18,702 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:19,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:19,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:19,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:19,839 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:20,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:20,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:21,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:23,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:23,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:23,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:23,977 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:25,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:27,124 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:10:27,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:27,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:27,842 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:29,147 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:29,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:29,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:29,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:31,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:32,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:32,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:33,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:34,131 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:10:34,220 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:34,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:34,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:34,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:34,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:34,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:35,845 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:36,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:36,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:37,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:39,308 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:39,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:39,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:39,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:41,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:41,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:41,847 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:43,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:44,387 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:44,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:44,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:45,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:46,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:46,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:47,849 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:48,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:48,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:49,462 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:49,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:49,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:49,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:49,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:51,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:10:51,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:51,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:51,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:52,072 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:10:52,072 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:10:52,073 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:10:52,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:52,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:10:53,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:53,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:53,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:54,549 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:55,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:56,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:56,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:57,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:10:59,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:59,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:59,633 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:59,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:01,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:01,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:01,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:03,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:04,132 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:11:04,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:04,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:04,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:04,904 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:11:04,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:04,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:04,938 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:07,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:08,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:09,939 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:10,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:10,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:12,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:12,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:14,940 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:15,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:15,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:17,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:17,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:17,858 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:19,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:19,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:19,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:19,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:19,859 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:20,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:21,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:22,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:22,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:23,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:24,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:24,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:25,081 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:25,861 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:26,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:26,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:27,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:29,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:29,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:29,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:30,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:31,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:31,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:31,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:33,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:33,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:33,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:34,133 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:11:34,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:34,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:34,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:35,222 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:35,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:36,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:36,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:37,865 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:38,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:38,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:39,865 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:40,312 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:41,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:42,611 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:11:42,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:42,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:43,867 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:45,395 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:45,409 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:11:45,409 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:11:45,410 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:11:45,410 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:11:45,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:45,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:45,867 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:45,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:11:46,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:47,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:47,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:47,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:49,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:49,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:49,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:49,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:49,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:49,869 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:50,506 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:51,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:52,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:52,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:53,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:54,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:54,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:55,582 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:55,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:56,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:56,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:57,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:59,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:11:59,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:59,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:00,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:01,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:02,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:02,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:03,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:04,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:04,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:04,135 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:12:04,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:04,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:04,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:05,739 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:05,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:07,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:07,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:07,875 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:09,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:09,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:09,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:10,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:11,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:11,194 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:11,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:13,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:14,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:14,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:15,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:15,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:16,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:16,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:17,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:19,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:19,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:19,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:19,982 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:12:20,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:21,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:21,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:21,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:23,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:23,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:23,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:25,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:25,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:25,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:26,184 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:27,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:27,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:12:27,918 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:12:27,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:12:27,919 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:12:28,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:28,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:28,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:28,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:12:29,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:30,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:30,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:31,338 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:31,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:32,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:32,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:33,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:34,136 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:12:34,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:34,801 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:34,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:35,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:35,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:35,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:36,496 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:37,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:37,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:37,886 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:39,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:40,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:40,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:41,661 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:41,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:42,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:42,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:43,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:45,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:45,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:45,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:46,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:47,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:47,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:47,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:49,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:49,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:49,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:49,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:49,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:49,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:51,891 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:51,975 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:52,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:52,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:53,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:54,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:54,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:55,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:57,124 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:57,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:12:58,282 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:12:58,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:58,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:59,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:01,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:01,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:01,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:02,209 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:03,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:03,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:03,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:04,138 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:13:04,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:04,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:04,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:05,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:05,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:05,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:07,283 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:07,896 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:08,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:08,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:09,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:10,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:10,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:10,680 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:13:10,682 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:13:10,682 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:13:10,682 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:13:10,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:13:11,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:12,345 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:12,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:12,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:12,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:13,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:15,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:15,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:15,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:17,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:17,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:17,466 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:17,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:19,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:19,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:19,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:20,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:20,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:21,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:22,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:22,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:22,521 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:23,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:25,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:25,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:25,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:27,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:27,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:27,584 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:27,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:29,904 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:30,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:30,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:32,584 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:32,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:32,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:33,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:34,140 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:13:34,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:34,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:34,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:35,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:36,502 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:13:36,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:36,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:37,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:38,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:39,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:39,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:39,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:41,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:41,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:41,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:43,275 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:43,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:43,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:43,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:45,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:46,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:46,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:47,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:48,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:48,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:48,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:49,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:49,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:49,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:49,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:50,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:50,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:51,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:53,619 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:53,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:54,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:54,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:55,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:56,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:56,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:57,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:13:57,086 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:13:57,087 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:13:57,087 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:13:57,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:13:57,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:13:58,792 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:59,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:59,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:59,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:01,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:01,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:01,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:03,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:03,946 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:04,142 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:14:04,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:04,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:04,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:04,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:04,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:05,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:06,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:06,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:07,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:08,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:08,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:09,152 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:09,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:11,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:11,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:11,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:13,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:14,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:15,296 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:14:15,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:15,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:15,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:17,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:17,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:17,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:19,473 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:19,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:19,802 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:19,920 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:20,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:20,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:21,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:22,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:22,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:23,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:24,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:24,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:24,657 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:25,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:27,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:27,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:27,923 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:29,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:29,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:29,767 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:29,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:31,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:31,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:31,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:33,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:34,143 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:14:34,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:34,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:34,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:34,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:34,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:35,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:35,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:36,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:36,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:37,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:38,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:38,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:39,928 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:39,941 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:14:39,942 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:14:39,943 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:14:39,943 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:14:40,928 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:14:40,944 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:41,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:41,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:41,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:43,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:43,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:43,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:45,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:45,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:45,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:46,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:47,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:48,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:48,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:49,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:49,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:49,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:49,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:51,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:51,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:52,765 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:14:52,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:52,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:53,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:54,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:54,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:55,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:56,345 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:57,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:57,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:57,935 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:14:59,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:59,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:59,936 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:01,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:01,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:01,936 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:01,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:03,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:04,146 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:15:04,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:04,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:04,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:04,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:05,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:05,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:06,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:07,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:07,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:07,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:09,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:09,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:09,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:11,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:11,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:11,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:12,374 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:13,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:14,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:14,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:15,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:16,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:16,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:17,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:17,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:19,500 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:15:19,501 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:15:19,502 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:15:19,503 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:15:19,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:19,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:19,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:19,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:19,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:15:20,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:20,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:21,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:22,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:22,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:23,185 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:24,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:24,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:25,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:27,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:27,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:27,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:28,313 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:29,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:31,271 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:15:31,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:31,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:31,947 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:33,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:33,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:33,376 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:33,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:34,148 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:15:34,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:34,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:34,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:35,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:36,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:36,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:37,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:38,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:38,477 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:38,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:39,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:41,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:41,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:41,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:43,555 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:43,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:43,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:43,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:45,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:45,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:45,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:47,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:48,618 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:48,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:48,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:49,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:49,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:49,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:49,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:50,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:50,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:51,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:52,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:52,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:53,674 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:53,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:55,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:55,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:55,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:57,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:15:57,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:57,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:59,486 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:59,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:00,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:00,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:01,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:03,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:03,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:03,579 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:16:03,580 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:16:03,581 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:16:03,582 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:16:03,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:03,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:16:04,149 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:16:04,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:04,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:04,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:04,957 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:05,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:05,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:05,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:07,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:08,978 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:16:09,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:09,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:09,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:10,335 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:11,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:12,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:12,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:13,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:14,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:14,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:15,434 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:15,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:17,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:17,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:17,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:19,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:19,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:19,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:19,803 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:19,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:19,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:20,497 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:21,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:21,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:21,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:23,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:24,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:24,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:25,572 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:25,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:26,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:26,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:27,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:28,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:28,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:29,968 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:30,651 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:31,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:31,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:32,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:33,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:34,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:34,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:34,150 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:16:34,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:34,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:34,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:35,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:35,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:36,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:36,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:37,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:39,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:39,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:39,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:40,790 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:41,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:41,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:41,973 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:43,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:43,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:43,974 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:45,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:45,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:16:45,904 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:16:45,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:16:45,905 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:16:45,974 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:45,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:16:47,022 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:16:47,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:47,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:48,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:49,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:49,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:49,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:49,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:50,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:50,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:50,972 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:51,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:52,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:52,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:53,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:55,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:55,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:55,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:56,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:57,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:57,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:57,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:16:59,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:59,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:59,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:01,107 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:01,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:02,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:02,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:03,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:04,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:04,152 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:17:04,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:04,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:04,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:04,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:05,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:06,171 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:07,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:07,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:07,984 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:09,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:09,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:09,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:11,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:11,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:11,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:11,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:13,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:14,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:14,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:15,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:16,356 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:16,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:16,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:17,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:18,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:18,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:19,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:19,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:19,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:21,419 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:21,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:21,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:21,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:23,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:25,137 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:17:25,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:25,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:25,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:26,486 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:27,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:27,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:27,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:28,190 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:17:28,191 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:17:28,192 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:17:28,192 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:17:28,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:17:29,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:30,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:30,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:31,559 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:31,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:32,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:32,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:33,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:34,153 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:17:34,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:34,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:34,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:35,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:35,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:35,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:36,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:37,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:37,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:39,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:39,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:41,977 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:42,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:42,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:44,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:44,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:46,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:46,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:46,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:49,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:49,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:49,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:49,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:49,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:50,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:51,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:51,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:52,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:52,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:54,003 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:54,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:54,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:56,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:56,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:56,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:58,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:17:58,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:59,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:59,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:00,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:02,006 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:03,381 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:18:03,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:03,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:03,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:04,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:04,155 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:18:04,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:04,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:04,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:05,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:05,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:06,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:08,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:08,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:08,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:08,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:10,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:10,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:10,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:12,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:12,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:12,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:13,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:14,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:15,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:15,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:16,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:17,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:17,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:18,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:19,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:19,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:19,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:19,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:20,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:20,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:22,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:23,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:18:23,121 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:18:23,121 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:18:23,122 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:18:23,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:23,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:24,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:24,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:18:24,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:25,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:25,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:26,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:27,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:27,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:28,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:29,169 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:30,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:30,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:30,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:32,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:32,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:32,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:34,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:34,157 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:18:34,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:34,804 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:34,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:35,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:35,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:35,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:36,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:37,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:37,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:38,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:40,021 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:40,994 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:41,351 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:18:41,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:41,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:42,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:44,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:44,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:44,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:46,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:46,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:46,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:46,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:48,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:48,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:48,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:49,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:49,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:49,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:50,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:51,143 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:51,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:51,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:52,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:53,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:53,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:55,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:55,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:55,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:56,212 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:57,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:18:58,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:58,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:59,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:00,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:00,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:01,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:01,285 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:02,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:02,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:03,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:04,159 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:19:04,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:04,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:04,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:05,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:05,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:19:05,367 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:19:05,368 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:19:05,368 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:19:06,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:19:06,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:06,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:06,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:07,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:08,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:08,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:08,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:09,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:11,032 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:11,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:11,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:11,440 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:13,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:13,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:13,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:15,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:15,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:15,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:16,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:17,034 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:18,822 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:19:18,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:18,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:19,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:19,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:19,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:21,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:21,585 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:21,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:21,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:23,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:23,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:23,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:25,037 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:25,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:25,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:26,669 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:27,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:28,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:29,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:29,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:31,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:31,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:31,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:31,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:33,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:33,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:33,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:34,160 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:19:34,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:34,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:34,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:35,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:36,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:36,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:36,835 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:37,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:38,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:38,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:39,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:40,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:40,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:41,043 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:41,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:43,044 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:43,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:43,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:45,044 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:45,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:45,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,001 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:47,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:47,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,699 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:19:47,700 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:19:47,700 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:19:47,701 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:19:48,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:19:49,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:49,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:49,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:49,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:49,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:49,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:50,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:51,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:52,057 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:52,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:52,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:53,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:55,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:56,108 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:19:56,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:56,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:57,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:19:57,132 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:58,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:58,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:59,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:01,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:01,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:01,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:02,203 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:03,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:03,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:03,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:04,162 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:20:04,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:04,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:04,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:05,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:05,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:05,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:07,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:07,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:08,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:08,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:09,054 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:10,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:10,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:11,054 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:12,380 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:12,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:12,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:13,055 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:15,056 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:15,828 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:15,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:17,057 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:17,455 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:17,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:17,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:19,057 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:19,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:19,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:19,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:20,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:20,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:21,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:22,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:22,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:22,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:23,059 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:24,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:24,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:25,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:27,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:27,592 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:28,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:28,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:29,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:29,992 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:20:29,994 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:20:29,994 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:20:29,995 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:20:30,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:20:30,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:30,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:31,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:32,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:32,665 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:33,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:34,164 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:20:34,243 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:20:34,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:34,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:34,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:34,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:34,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:35,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:37,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:37,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:37,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:37,749 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:39,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:39,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:39,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:41,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:41,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:41,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:42,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:43,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:44,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:44,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:45,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:46,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:46,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:47,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:47,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:48,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:48,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:49,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:49,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:49,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:49,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:51,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:51,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:51,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:52,940 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:53,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:53,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:53,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:55,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:56,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:56,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:57,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:20:58,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:58,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:58,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:59,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:00,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:00,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:01,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:02,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:02,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:03,055 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:03,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:04,165 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:21:04,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:04,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:04,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:05,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:05,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:05,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:07,078 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:08,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:08,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:08,137 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:09,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:11,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:11,791 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:21:11,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:11,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:12,217 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:21:12,218 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:21:12,218 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:21:12,219 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:21:13,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:13,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:21:13,220 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:14,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:14,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:15,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:15,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:17,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:17,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:17,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:18,274 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:19,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:19,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:19,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:19,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:19,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:19,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:21,083 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:22,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:22,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:23,084 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:23,343 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:24,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:24,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:25,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:26,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:26,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:27,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:28,410 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:29,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:29,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:29,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:31,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:31,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:31,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:33,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:33,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:34,167 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:21:34,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:34,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:34,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:34,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:34,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:35,088 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:36,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:36,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:37,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:38,564 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:38,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:38,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:39,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:41,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:41,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:41,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:43,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:43,629 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:43,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:43,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:45,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:46,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:46,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:47,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:48,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:49,094 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:49,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:49,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:49,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:50,512 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:21:50,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:50,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:51,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:53,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:53,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:53,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:53,766 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:54,479 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:21:54,481 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:21:54,481 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:21:54,482 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:21:55,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:55,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:21:55,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:55,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:56,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:57,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:21:58,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:58,838 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:58,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:59,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:00,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:00,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:01,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:03,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:03,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:04,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:04,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:04,169 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:22:04,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:04,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:04,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:05,100 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:06,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:06,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:07,101 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:08,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:08,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:08,993 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:09,101 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:11,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:11,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:11,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:13,103 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:13,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:13,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:14,049 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:15,103 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:15,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:15,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:17,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:18,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:18,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:19,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:19,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:19,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:19,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:19,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:20,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:20,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:21,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:22,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:22,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:23,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:24,186 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:24,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:24,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:25,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:27,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:28,797 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:22:28,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:28,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:29,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:29,242 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:31,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:31,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:31,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:33,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:33,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:33,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:34,174 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:22:34,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:34,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:34,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:35,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:35,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:36,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:36,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:36,729 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:22:36,730 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:22:36,731 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:22:36,731 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:22:37,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:37,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:22:38,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:38,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:38,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:39,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:40,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:40,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:40,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:41,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:43,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:43,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:44,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:45,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:45,299 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:46,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:46,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:47,116 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:48,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:48,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:49,117 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:49,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:49,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:49,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:51,009 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:51,118 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:51,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:51,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:53,118 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:53,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:53,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:55,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:55,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:55,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:56,109 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:57,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:22:58,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:58,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:59,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:00,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:00,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:01,121 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:01,173 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:02,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:02,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:03,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:04,176 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:23:04,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:04,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:04,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:05,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:06,063 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:23:06,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:06,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:06,251 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:07,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:09,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:09,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:09,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:11,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:11,141 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:11,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:11,330 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:13,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:13,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:13,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:15,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:16,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:16,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:16,385 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:17,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:18,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:18,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:19,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:19,791 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:23:19,792 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:23:19,792 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:23:19,792 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:23:19,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:19,806 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:19,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:20,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:23:21,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:21,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:21,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:21,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:22,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:23,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:23,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:23,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:25,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:25,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:25,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:26,545 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:27,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:28,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:28,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:29,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:30,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:30,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:31,131 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:31,604 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:32,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:32,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:33,131 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:34,177 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:23:34,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:34,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:34,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:35,132 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:35,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:35,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:36,667 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:37,133 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:37,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:37,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:39,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:39,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:39,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:41,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:41,747 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:43,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:43,717 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:23:43,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:43,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:45,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:46,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:46,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:46,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:47,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:48,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:48,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:49,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:49,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:49,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:49,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:50,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:50,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:51,138 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:51,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:53,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:53,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:53,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:55,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:55,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:55,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:56,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:57,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:23:59,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:59,018 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:59,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:01,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:01,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:01,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:02,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:02,049 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:24:02,050 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:24:02,050 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:24:02,050 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:24:02,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:24:03,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:03,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:03,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:04,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:04,179 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:24:04,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:04,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:04,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:05,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:06,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:06,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:08,021 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:08,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:08,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:10,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:10,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:13,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:13,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:13,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:15,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:15,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:17,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:17,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:18,520 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:19,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:19,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:19,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:19,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:21,147 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:21,208 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:24:21,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:21,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:23,148 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:23,948 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:24,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:24,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:25,148 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:26,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:26,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:27,149 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:28,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:28,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:29,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:29,150 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:31,150 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:31,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:31,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:33,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:33,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:33,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:34,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:34,180 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:24:34,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:34,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:34,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:35,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:36,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:36,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:37,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:38,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:38,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:39,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:39,178 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:40,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:40,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:41,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:42,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:42,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:43,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:44,236 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:45,156 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:45,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:45,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:47,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:47,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:47,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:49,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:49,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:49,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:49,807 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:49,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:49,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:49,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:51,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:51,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:51,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:53,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:54,394 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:54,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:54,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:55,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:56,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:24:56,105 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:24:56,106 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:24:56,106 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:24:56,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:24:57,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:58,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:58,689 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:24:58,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:58,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:59,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:24:59,462 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:00,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:00,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:01,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:03,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:03,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:03,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:04,182 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:25:04,533 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:04,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:04,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:04,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:05,163 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:05,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:05,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:07,164 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:07,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:07,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:09,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:09,639 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:09,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:09,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:11,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:12,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:12,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:13,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:14,715 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:14,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:14,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:15,167 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:16,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:16,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:17,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:19,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:19,793 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:19,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:19,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:19,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:19,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:19,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:21,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:21,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:21,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:23,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:23,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:23,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:24,858 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:25,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:27,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:27,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:27,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:29,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:29,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:29,172 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:29,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:31,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:31,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:31,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:33,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:34,186 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:25:34,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:34,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:34,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:34,833 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:25:34,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:34,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:34,940 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:35,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:37,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:37,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:38,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:38,429 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:25:38,430 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:25:38,430 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:25:38,431 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:25:39,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:39,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:25:40,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:40,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:40,105 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:40,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:41,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:42,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:42,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:43,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:45,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:45,185 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:45,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:45,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:47,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:47,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:47,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:49,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:49,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:49,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:49,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:50,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:50,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:50,832 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:51,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:52,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:52,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:53,181 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:54,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:54,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:55,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:56,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:57,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:57,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:57,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:59,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:25:59,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:59,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:01,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:01,396 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:01,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:01,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:03,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:04,186 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:26:04,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:04,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:04,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:04,808 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:04,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:05,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:06,491 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:06,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:06,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:07,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:08,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:08,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:09,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:11,189 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:11,577 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:12,535 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:26:12,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:12,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:13,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:15,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:15,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:15,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:16,647 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:17,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:17,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:17,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:19,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:19,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:19,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:19,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:19,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:19,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:20,745 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:26:20,747 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:26:20,747 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:26:20,747 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:26:21,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:21,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:26:21,748 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:22,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:22,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:22,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:23,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:24,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:24,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:25,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:26,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:27,194 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:27,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:27,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:29,195 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:29,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:29,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:31,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:31,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:31,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:31,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:33,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:34,188 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:26:34,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:34,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:34,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:35,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:35,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:35,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:36,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:37,089 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:37,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:37,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:39,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:40,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:40,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:41,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:42,051 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:42,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:42,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:43,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:45,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:45,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:45,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:47,127 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:47,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:47,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:47,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:49,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:49,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:49,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:49,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:51,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:52,208 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:52,271 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:26:52,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:52,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:53,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:54,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:54,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:55,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:57,207 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:57,279 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:57,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:57,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:59,208 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:26:59,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:59,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:01,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:02,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:02,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:02,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:03,047 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:27:03,049 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:27:03,049 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:27:03,049 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:27:03,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:03,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:27:04,190 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:27:04,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:04,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:04,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:04,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:04,809 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:04,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:05,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:06,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:06,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:07,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:07,421 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:09,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:09,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:09,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:11,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:11,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:11,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:12,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:13,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:13,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:13,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:15,214 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:16,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:16,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:17,214 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:17,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:18,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:18,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:19,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:19,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:19,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:19,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:20,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:20,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:21,216 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:22,617 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:23,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:24,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:24,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:25,218 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:26,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:26,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:27,218 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:27,694 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:29,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:29,823 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:27:29,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:29,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:31,220 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:32,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:32,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:32,785 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:33,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:34,192 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:27:34,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:34,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:34,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:34,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:34,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:35,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:37,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:37,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:37,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:37,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:39,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:39,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:39,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:41,223 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:42,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:42,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:42,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:43,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:44,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:44,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:45,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:45,289 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:27:45,290 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:27:45,290 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:27:45,291 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:27:46,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:46,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:27:47,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:47,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:47,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:47,973 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:49,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:49,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:49,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:49,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:50,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:50,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:51,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:52,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:52,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:53,054 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:53,227 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:54,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:54,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:55,227 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:57,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:57,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:57,228 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:58,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:59,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:27:59,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:59,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:01,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:01,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:01,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:03,210 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:03,230 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:04,193 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:28:04,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:04,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:04,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:04,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:04,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:05,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:07,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:08,263 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:28:08,280 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:08,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:08,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:09,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:10,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:10,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:11,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:12,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:12,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:13,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:13,352 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:15,234 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:15,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:15,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:17,234 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:17,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:17,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:18,945 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:19,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:19,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:19,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:19,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:20,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:20,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:21,236 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:22,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:22,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:23,236 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:24,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:24,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:24,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:25,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:27,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:27,399 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:28:27,401 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:28:27,401 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:28:27,402 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:28:27,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:27,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:28,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:28,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:28:29,090 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:29,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:29,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:29,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:31,239 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:31,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:31,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:33,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:34,146 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:34,195 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:28:34,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:34,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:34,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:34,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:34,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:35,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:36,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:36,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:37,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:38,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:39,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:39,225 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:39,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:41,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:42,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:42,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:43,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:44,300 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:45,245 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:45,864 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:28:45,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:45,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:47,245 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:47,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:47,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:49,246 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:49,377 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:49,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:49,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:49,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:49,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:49,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:51,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:52,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:53,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:53,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:54,436 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:55,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:55,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:55,248 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:57,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:58,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:58,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:59,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:28:59,505 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:00,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:00,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:01,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:02,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:02,141 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:03,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:04,197 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:29:04,567 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:04,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:04,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:04,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:05,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:05,251 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:05,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:07,251 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:07,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:07,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:09,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:09,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:09,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:09,643 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:09,656 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:29:09,658 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:29:09,658 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:29:09,659 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:29:10,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:29:11,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:12,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:12,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:12,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:13,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:14,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:14,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:14,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:15,254 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:16,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:16,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:17,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:19,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:19,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:19,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:19,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:19,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:19,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:19,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:21,256 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:23,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:23,615 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:29:23,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:23,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:24,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:25,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:25,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:25,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:27,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:28,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:28,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:29,259 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:29,948 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:30,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:30,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:31,260 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:32,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:32,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:33,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:34,199 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:29:34,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:34,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:34,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:34,999 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:35,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:35,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:35,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:37,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:37,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:37,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:39,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:39,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:39,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:40,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:41,263 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:42,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:42,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:43,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:44,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:44,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:45,173 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:45,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:46,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:46,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:47,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:49,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:49,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:49,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:50,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:50,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:50,475 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:51,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:51,959 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:29:51,962 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:29:51,962 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:29:51,963 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:29:52,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:52,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:52,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:29:53,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:54,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:54,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:54,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:55,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:56,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:56,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:56,334 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:57,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:59,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:29:59,959 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:29:59,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:59,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:01,270 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:01,407 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:03,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:03,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:03,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:04,201 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:30:04,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:04,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:04,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:05,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:05,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:05,272 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:06,510 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:07,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:07,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:07,272 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:09,273 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:10,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:10,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:11,274 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:11,593 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:12,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:12,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:13,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:14,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:14,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:15,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:16,673 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:17,276 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:17,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:17,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:19,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:19,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:19,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:19,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:19,810 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:19,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:21,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:21,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:21,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:21,756 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:23,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:24,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:24,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:25,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:26,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:26,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:27,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:27,419 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:28,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:28,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:29,281 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:31,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:31,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:31,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:32,701 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:33,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:33,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:34,202 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:30:34,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:34,812 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:34,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:37,491 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:30:37,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:37,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:38,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:40,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:40,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:42,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:42,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:43,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:45,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:45,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:45,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:45,576 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:30:45,577 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:30:45,578 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:30:45,578 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:30:46,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:46,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:30:47,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:30:48,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:48,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:48,581 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:49,810 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:49,811 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:50,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:50,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:52,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:52,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:53,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:55,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:55,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:57,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:57,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:59,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,291 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:31:02,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:02,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:04,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:04,204 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:31:04,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:04,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:04,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:04,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:04,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:06,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:07,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:09,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:09,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:09,946 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:11,295 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:31:11,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:11,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,162 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:31:15,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:18,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:18,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:19,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:19,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:19,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:20,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:20,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:21,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:22,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:22,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:25,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:31:25,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:25,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:26,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:27,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:27,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:29,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:29,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:31,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:32,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:32,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:34,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:31:34,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:34,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:34,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:34,812 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:34,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:37,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:37,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:37,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:39,304 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:31:39,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:39,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:42,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:42,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:42,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:44,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:44,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:47,320 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:47,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:48,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:49,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:50,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:50,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:51,308 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:31:52,633 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:53,654 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:31:53,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:53,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:56,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:56,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:57,635 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:58,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:58,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:01,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:01,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:02,636 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:03,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:03,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:04,208 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:32:04,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:04,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:04,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:05,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:32:05,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:05,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:07,981 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:08,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:08,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:10,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:10,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:12,982 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:13,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:13,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:15,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:15,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:17,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:17,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:18,478 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:19,317 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:32:19,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:19,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:19,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:20,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:20,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:22,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:22,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:24,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:25,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:25,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:27,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:27,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:29,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:31,320 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:32:31,772 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:32:31,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:31,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:34,213 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:32:34,218 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:34,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:34,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:34,814 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:34,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:34,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:36,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:36,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:38,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:38,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:40,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:41,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:41,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:43,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:43,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:45,293 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:45,325 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:32:45,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:45,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:48,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:48,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:49,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:49,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:50,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:50,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:51,019 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:53,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:53,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:55,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:55,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:56,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:57,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:57,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:59,329 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:33:00,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:00,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:01,666 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:02,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:02,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:04,211 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:33:04,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:04,813 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:04,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:04,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:04,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:06,999 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:08,488 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:33:08,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:08,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:11,333 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:33:11,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:11,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:12,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:13,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:13,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:15,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:15,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:17,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:18,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:18,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:19,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:19,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:19,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:20,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:20,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:22,597 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:22,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:22,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:25,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:33:25,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:25,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:27,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:27,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:28,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:30,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:30,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:32,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:32,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:33,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:34,213 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:33:34,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:34,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:34,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:34,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:34,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:38,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:38,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:38,788 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:39,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:33:40,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:40,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:42,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:42,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:43,789 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:45,713 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:33:45,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:45,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:48,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:48,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:48,790 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:49,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:49,814 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:49,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:50,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:50,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:52,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:52,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:54,288 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:54,348 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:33:55,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:55,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:57,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:57,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:59,289 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:59,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:59,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:02,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:02,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:04,215 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:34:04,552 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:04,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:04,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:04,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:04,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:04,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:06,352 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:34:06,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:06,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:09,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:09,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:10,002 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:11,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:11,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:13,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:13,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:15,003 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:16,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:16,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:18,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:18,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:19,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:19,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:19,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:20,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:34:20,985 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:22,374 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:34:22,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:22,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:25,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:25,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:25,986 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:27,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:27,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:29,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:29,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:30,987 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:32,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:32,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:34,217 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:34:34,361 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:34:34,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:34,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:34,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:34,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:34,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:35,993 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:36,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:36,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:39,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:39,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:40,994 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:41,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:41,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:43,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:43,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:46,365 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:34:46,653 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:46,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:46,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:48,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:48,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:49,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:49,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:49,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:51,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:51,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:52,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:53,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:53,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:55,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:55,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:57,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:59,524 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:34:59,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:59,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:00,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:35:02,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:02,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:02,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:04,219 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:35:04,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:04,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:04,814 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:04,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:04,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:06,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:06,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:07,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:09,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:09,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:11,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:11,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:12,370 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:13,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:13,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:14,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:35:16,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:16,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:17,370 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:18,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:18,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:19,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:19,815 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:20,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:20,953 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:22,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:22,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:23,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:25,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:25,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:26,378 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:35:27,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:27,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:28,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:30,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:30,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:33,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:33,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:33,767 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:34,220 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:35:34,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:34,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:34,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:36,756 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:35:36,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:36,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:38,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:39,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:39,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:40,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:35:41,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:41,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:43,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:43,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:43,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:46,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:46,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:48,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:49,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:49,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:49,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:49,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:49,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:51,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:51,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:52,385 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:35:54,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:54,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:54,203 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:56,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:56,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:58,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:58,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:59,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:01,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:01,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:03,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:03,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:04,206 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:04,222 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:36:04,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:04,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:04,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:05,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:05,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:06,390 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:07,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:07,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:09,957 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:10,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:10,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:13,915 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:36:13,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:13,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:14,957 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:15,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:15,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:19,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:19,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:19,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:19,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:20,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:20,393 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:21,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:21,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:23,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:23,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:25,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:26,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:26,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:28,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:28,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:30,026 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:30,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:30,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:32,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:33,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:33,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:34,224 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:36:34,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:34,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:34,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:35,412 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:35,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:35,997 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:37,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:37,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:40,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:40,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:41,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:42,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:42,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:42,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:44,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:44,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:47,341 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:47,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:47,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:48,402 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:49,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:49,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:49,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:51,440 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:36:51,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:51,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:52,805 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:54,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:54,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:56,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:56,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:58,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:36:58,593 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:58,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:58,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:01,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:01,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:03,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:03,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:03,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:04,226 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:37:04,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:04,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:04,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:06,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:06,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:08,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:08,607 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:08,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:10,408 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:37:10,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:10,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,653 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:15,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:15,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:18,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:18,654 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:18,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:19,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:19,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:20,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:20,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:23,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:23,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:23,956 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:24,412 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:37:25,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:25,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:28,957 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:29,531 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:37:29,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:29,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:32,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:32,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:33,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:34,227 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:37:34,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:34,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:34,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:34,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:34,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:36,416 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:37:36,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:36,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:39,285 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:39,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:39,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:41,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:41,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:43,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:43,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:44,286 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:46,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:46,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:48,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:48,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:49,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:49,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:49,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:49,987 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:50,421 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:37:51,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:51,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:53,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:53,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:54,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:56,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:56,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:58,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:58,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:59,990 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:01,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:01,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:02,427 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:38:03,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:03,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:04,228 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:38:04,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:04,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:04,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:05,946 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:06,878 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:38:06,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:06,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:09,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:09,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:10,946 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:11,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:11,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:13,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:13,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:16,237 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:16,431 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:38:16,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:16,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:18,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:18,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:19,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:19,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:20,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:20,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:21,315 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:22,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:22,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:25,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:25,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:26,315 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:27,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:27,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:29,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:29,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:30,436 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:38:31,753 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:31,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:31,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:34,230 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:38:34,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:34,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:34,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:34,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:34,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:36,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:36,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:36,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:38,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:38,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:41,977 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:42,441 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:38:42,516 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:38:42,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:42,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:45,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:45,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:46,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:47,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:47,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:49,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:49,816 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:49,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:50,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:50,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:52,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:52,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:52,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:54,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:54,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:56,444 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:38:57,331 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:57,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:57,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:59,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:59,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:02,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:02,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:02,331 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:04,232 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:39:04,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:04,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:04,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:05,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:05,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:07,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:07,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:07,585 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:09,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:09,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:10,450 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:39:12,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:12,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:12,593 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:14,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:14,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:16,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:16,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:17,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:19,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:19,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:19,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:22,405 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:39:22,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:22,453 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:39:22,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:22,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:24,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:24,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:26,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:26,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:27,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:28,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:28,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:31,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:31,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:32,979 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:34,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:34,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:34,234 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:39:34,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:34,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:34,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:36,458 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:39:37,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:37,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:38,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:39,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:39,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:41,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:41,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:43,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:44,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:44,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:46,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:46,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:48,496 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:48,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:48,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:49,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:49,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:50,462 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:39:51,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:51,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:53,653 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:53,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:53,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:55,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:55,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:58,655 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:59,427 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:39:59,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:59,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:02,466 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:40:02,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:02,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:03,851 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:04,236 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:40:04,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:04,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:04,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:04,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:04,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:06,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:06,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:08,974 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:09,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:09,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:11,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:11,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:13,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:13,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:14,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:16,469 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:40:16,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:16,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:18,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:18,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:19,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:19,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:19,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:20,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:20,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:21,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:23,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:23,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:25,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:25,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:26,182 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:27,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:27,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:28,473 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:40:31,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:31,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:32,174 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:33,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:33,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:34,237 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:40:34,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:34,817 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:34,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:36,859 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:40:36,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:36,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:37,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:39,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:40,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,477 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:40:43,269 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:44,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:44,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:47,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:47,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:48,270 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:49,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:49,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:49,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:49,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:51,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:51,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:54,265 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:54,481 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:40:54,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:54,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:56,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:56,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:59,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:59,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:59,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:01,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:01,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:03,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:04,239 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:41:04,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:04,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:04,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:04,972 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:04,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:07,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:07,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:08,485 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:41:09,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:09,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:10,785 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:12,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:12,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:15,786 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:15,922 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:41:15,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:15,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:18,018 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:18,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:19,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:19,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:19,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:20,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:20,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:21,169 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:22,489 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:41:23,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:23,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:25,120 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:25,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:26,170 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:27,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:27,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:30,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:30,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:31,171 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:32,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:32,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:34,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:34,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:34,240 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:41:34,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:34,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:34,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:36,499 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:41:36,960 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:37,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:37,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:39,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:39,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:41,961 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:42,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:42,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:44,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:44,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,962 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:48,502 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:41:49,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:49,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:49,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:49,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:52,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:53,330 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:41:53,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:53,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:55,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:55,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:57,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:58,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:58,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:00,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:00,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:02,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:02,506 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:42:02,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:02,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:04,242 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:42:04,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:04,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:04,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:05,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:05,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:07,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:07,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:07,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:09,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:09,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:12,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:12,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:12,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:14,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:14,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:16,510 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:42:16,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:16,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:18,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:19,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:19,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:19,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:19,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:21,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:21,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:23,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:24,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:24,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:26,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:26,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:28,513 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:42:29,333 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:30,591 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:42:30,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:30,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:33,482 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:33,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:34,244 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:42:34,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:34,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:34,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:34,995 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:35,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:35,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:38,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:38,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:39,995 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:40,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:40,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:42,517 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:42:43,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:43,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:45,671 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:45,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:45,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:48,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:48,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:49,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:49,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:50,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:50,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:50,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:53,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:53,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:54,521 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:42:55,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:55,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:56,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:58,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:58,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:00,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:00,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:01,359 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:03,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:03,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:04,246 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:43:04,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:04,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:04,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:05,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:05,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:07,059 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:08,525 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:43:09,688 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:43:10,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:10,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:12,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:13,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:14,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:16,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:16,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:17,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:19,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:19,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:19,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:19,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:21,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:21,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:22,530 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:43:23,457 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:24,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:24,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:26,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:26,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:28,458 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:29,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:29,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:31,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:31,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:33,459 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:34,249 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:43:34,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:34,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:34,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:34,818 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:34,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:36,540 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:43:36,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:36,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:39,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:39,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:39,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:41,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:41,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:44,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:44,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:45,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:47,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:48,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:49,361 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:49,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:49,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:50,545 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:43:52,358 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:43:52,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:52,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:55,126 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:55,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:55,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:57,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:57,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:00,127 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:00,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:00,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:02,548 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:44:02,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:02,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:04,250 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:44:04,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:04,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:04,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:04,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:05,975 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:08,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:08,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:10,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:10,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:10,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:12,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:12,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:15,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:15,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:16,553 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:44:16,849 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:17,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:17,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:19,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:19,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:19,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:19,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:19,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:21,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:22,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:22,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:24,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:24,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:26,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:27,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:27,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:28,557 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:44:31,161 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:44:31,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:31,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:32,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:34,251 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:44:34,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:34,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:34,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:34,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:34,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:36,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:36,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:37,948 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:38,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:38,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:41,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:41,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:43,393 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:43,563 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:44:45,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:45,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:47,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:47,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:48,394 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:49,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:49,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:49,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:49,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:52,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:52,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:53,571 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:54,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:54,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:56,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:56,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:57,567 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:44:58,773 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:59,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:59,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:01,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:01,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:03,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:03,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:03,774 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:04,253 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:45:04,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:04,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:06,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:06,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:08,953 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:09,570 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:45:10,332 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:45:10,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:10,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:12,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:12,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:13,953 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:15,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:15,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:17,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:17,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:18,954 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:19,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:19,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:19,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:19,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:19,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:22,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:22,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:23,575 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:45:24,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:24,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:24,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:26,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:26,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:29,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:29,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:29,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:31,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:31,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:33,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:33,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:34,140 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:34,255 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:45:34,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:34,819 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:34,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:36,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:36,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:37,580 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:45:38,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:38,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:40,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:40,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:40,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:44,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:44,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:45,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:47,807 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:45:47,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:47,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:49,583 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:45:49,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:49,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:49,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:49,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:50,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:52,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:52,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:54,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:54,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:55,051 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:56,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:56,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:59,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:59,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:00,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:02,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:02,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:03,587 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:46:04,257 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:46:04,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:04,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:05,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:05,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:06,016 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:07,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:07,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:09,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:09,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:11,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:12,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:12,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:14,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:14,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:15,590 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:46:16,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:16,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:16,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:19,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:19,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:19,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:19,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:19,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:21,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:21,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:21,984 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:25,472 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:46:25,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:25,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:26,989 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:28,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:28,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:29,595 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:46:30,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:30,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:32,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:32,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:32,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:34,258 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:46:34,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:34,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:34,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:34,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:34,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:37,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:37,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:37,970 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:39,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:39,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:42,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:43,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:43,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:43,599 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:46:45,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:45,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:48,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:48,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:48,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:49,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:50,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:50,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:52,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:52,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:54,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:55,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:55,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:57,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:57,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:57,603 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:46:59,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:59,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:59,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:03,102 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:47:03,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:03,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:04,260 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:47:04,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:04,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:04,992 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:06,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:06,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:08,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:08,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:09,607 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:47:10,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:10,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:10,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:12,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:12,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:15,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:15,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:15,524 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:17,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:17,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:19,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:19,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:19,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:19,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:19,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:21,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:22,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:22,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:23,611 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:47:24,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:24,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:26,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:27,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:27,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:29,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:29,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:31,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:31,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:31,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:34,262 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:47:34,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:34,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:34,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:34,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:34,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:36,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:36,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:36,589 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:37,615 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:47:40,367 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:47:40,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:40,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:41,590 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:43,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:43,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:45,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:45,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:47,284 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:47,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:47,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:47,619 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:47:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:49,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:49,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:50,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:50,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:52,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:52,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:52,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:53,621 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:47:54,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:54,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:57,393 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:57,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:57,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:59,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:59,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:01,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:01,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:03,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:03,625 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:48:04,263 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:48:04,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:04,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:04,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:04,820 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:06,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:06,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:08,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:09,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:09,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:11,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:11,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:13,629 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:48:14,528 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:14,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:14,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:18,296 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:48:18,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:18,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:19,529 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:19,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:19,821 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:19,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:21,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:21,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:23,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:23,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:25,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:25,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:25,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:27,634 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:48:28,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:28,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:30,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:30,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:30,294 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:33,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:33,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:34,265 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:48:34,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:34,821 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:34,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:35,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:35,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:36,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:38,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:38,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:39,638 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:48:40,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:40,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:41,203 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:42,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:42,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:45,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:45,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:46,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:47,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:47,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:49,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:49,821 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:49,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:50,135 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:50,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:51,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:52,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:52,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:53,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:48:55,575 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:48:55,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:55,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:56,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:58,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:58,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:00,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:00,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:01,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:03,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:03,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:04,267 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:49:04,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:04,821 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:04,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:05,648 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:49:05,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:05,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:07,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:08,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:08,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:10,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:10,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:12,008 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:13,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:13,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:15,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:15,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:17,213 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:17,653 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:49:18,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:18,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:19,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:19,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:19,826 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:20,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:20,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:22,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:22,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:22,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:25,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:25,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:27,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:27,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:27,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:29,660 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:49:30,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:30,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:33,430 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:34,268 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:49:34,345 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:49:34,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:34,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:34,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:34,822 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:34,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:37,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:37,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:39,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:39,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:39,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:41,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:41,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:43,665 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:49:44,093 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:44,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:44,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:46,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:46,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:48,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:48,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:49,094 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:49,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:49,822 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:49,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:51,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:51,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:53,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:53,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:54,137 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:55,668 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:49:55,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:55,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:58,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:58,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:59,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:00,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:00,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:03,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:03,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:04,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:04,270 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:50:04,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:04,822 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:04,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:05,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:05,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:08,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:08,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:09,672 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:50:09,948 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:11,605 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:50:11,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:11,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:14,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:14,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:14,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:16,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:16,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:19,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:19,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:19,822 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:19,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:19,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:20,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:21,675 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:50:21,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:21,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:23,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:24,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:25,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:26,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:26,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:28,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:28,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:30,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:30,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:30,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:33,679 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:50:33,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:33,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:34,271 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:50:34,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:34,823 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:34,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:35,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:35,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:36,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:37,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:37,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:40,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:40,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:41,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:42,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:42,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:44,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:44,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:45,683 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:50:46,230 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:48,267 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:50:48,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:48,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:49,823 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:49,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:51,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:51,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:52,017 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:53,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:53,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:55,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:55,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:57,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:57,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:50:58,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:58,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:00,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:00,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:02,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:03,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:03,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:04,273 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:51:04,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:04,822 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:04,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:05,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:05,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:07,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:07,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:08,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:10,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:10,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:11,690 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:12,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:12,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:13,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:14,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:14,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:17,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:17,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:18,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:19,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:19,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:19,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:19,823 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:19,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:21,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:21,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:23,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:23,694 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:24,979 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:51:25,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:25,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:27,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:27,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:28,103 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:29,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:29,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:31,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:31,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:33,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:34,275 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:51:34,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:34,823 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:34,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:35,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:35,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:37,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:37,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:37,700 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:39,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:39,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:39,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:42,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:42,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:44,051 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:44,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:44,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:47,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:47,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:49,178 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:49,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:49,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:49,703 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:49,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:49,823 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:49,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:52,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:52,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:54,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:54,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:54,309 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:55,402 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:51:55,404 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:51:55,405 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:51:55,406 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:51:55,706 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:55,706 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:51:56,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:56,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:57,706 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:51:59,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:59,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:59,407 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:02,913 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:52:02,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:02,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:04,369 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:52:04,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:04,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:05,262 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:05,263 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:05,330 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:05,439 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:05,709 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:08,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:08,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:10,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:10,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:10,439 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:13,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:13,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:15,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:15,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:15,713 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:16,304 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:17,714 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:18,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:18,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:19,715 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:19,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:19,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:19,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:20,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:20,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:21,459 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:21,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:22,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:22,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:23,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:25,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:25,717 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:25,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:27,406 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:27,718 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:27,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:27,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:29,719 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:29,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:29,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:31,719 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:32,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:32,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:32,954 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:33,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:34,278 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:52:34,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:34,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:34,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:34,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:34,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:35,721 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:37,721 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:37,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:37,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:38,342 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:39,722 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:41,597 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:52:41,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:41,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:41,723 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:43,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:43,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:43,724 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:43,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:45,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:45,724 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:46,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:47,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:48,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:48,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:49,425 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:49,726 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:49,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:49,860 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:50,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:50,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:51,727 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:53,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:52:53,519 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:52:53,519 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:52:53,520 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:52:53,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:53,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:52:53,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:53,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:54,560 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:55,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:56,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:56,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:57,729 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:52:58,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:58,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:59,648 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:59,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:00,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:00,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:01,731 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:03,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:03,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:03,731 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:04,280 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:53:04,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:04,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:04,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:04,997 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:05,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:05,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:05,732 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:07,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:07,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:07,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:09,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:10,399 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:10,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:10,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:11,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:12,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:12,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:13,735 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:14,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:14,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:15,736 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:16,272 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:17,736 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:18,431 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:53:18,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:18,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:19,737 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:19,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:19,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:19,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:21,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:21,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:21,721 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:21,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:23,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:23,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:23,739 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:25,739 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:26,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:26,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:27,667 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:27,740 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:28,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:28,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:29,741 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:30,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:30,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:31,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:32,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:33,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:33,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:33,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:34,282 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:53:34,476 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:53:34,478 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:53:34,478 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:53:34,478 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:53:34,743 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:53:34,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:34,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:34,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:35,743 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:35,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:35,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:37,745 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:37,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:37,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:38,748 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:39,745 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:40,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:40,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:41,745 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:42,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:42,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:43,746 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:43,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:45,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:45,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:45,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:47,748 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:48,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:48,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:49,749 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:49,839 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:49,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:49,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:49,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:50,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:50,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:51,749 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:52,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:52,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:53,750 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:55,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:55,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:56,760 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:53:57,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:57,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:57,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:53:59,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:59,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:59,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:00,851 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:01,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:01,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:01,753 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:03,754 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:04,284 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:54:04,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:04,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:04,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:04,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:04,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:05,755 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:06,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:06,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:06,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:07,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:09,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:09,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:09,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:11,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:11,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:11,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:11,772 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:13,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:13,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:13,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:14,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:54:14,054 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:54:14,054 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:54:14,055 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:54:14,760 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:54:15,760 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:16,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:16,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:17,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:17,761 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:18,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:18,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:19,762 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:19,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:19,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:19,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:20,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:20,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:21,762 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:22,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:23,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:23,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:23,763 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:25,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:25,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:25,764 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:27,766 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:28,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:28,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:28,416 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:29,767 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:30,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:30,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:31,768 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:33,768 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:34,152 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:54:34,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:34,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:34,286 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:54:34,287 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:34,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:34,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:34,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:35,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:37,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:37,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:37,770 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:39,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:39,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:39,315 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:39,771 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:41,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:42,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:42,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:43,773 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:44,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:44,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:45,214 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:45,773 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:46,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:46,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:47,774 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:49,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:49,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:49,775 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:49,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:49,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:49,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:51,059 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:51,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:51,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:51,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:53,743 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:54:53,744 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:54:53,744 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:54:53,745 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:54:53,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:53,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:54:54,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:54,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:55,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:56,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:56,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:56,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:57,778 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:54:59,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:59,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:59,779 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:01,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:01,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:01,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:01,780 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:03,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:04,287 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:55:04,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:04,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:04,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:04,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:04,861 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:05,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:06,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:06,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:07,381 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:07,782 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:08,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:08,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:09,783 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:11,784 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:12,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:13,310 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:55:13,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:13,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:13,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:15,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:15,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:15,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:17,787 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:18,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:18,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:18,492 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:19,788 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:19,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:19,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:19,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:20,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:20,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:21,789 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:23,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:23,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:23,789 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:24,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:25,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:25,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:25,790 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:27,791 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:28,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:28,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:29,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:29,792 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:31,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:31,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:31,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:33,794 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:34,289 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:55:34,290 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:34,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:34,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:34,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:34,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:34,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:36,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:55:36,558 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:55:36,558 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:55:36,560 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:55:36,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:55:36,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:36,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:37,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:38,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:38,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:38,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:39,569 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:39,796 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:41,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:41,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:41,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:43,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:43,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:44,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:45,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:45,798 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:46,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:46,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:47,799 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:49,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:49,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:49,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:49,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:49,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:49,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:51,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:51,800 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:52,779 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:55:52,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:52,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:53,801 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:54,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:54,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:55,802 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:56,163 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:56,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:56,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:57,803 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:59,803 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:55:59,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:59,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:01,804 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:01,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:01,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:02,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:03,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:03,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:03,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:04,290 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:56:04,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:04,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:04,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:05,806 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:06,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:06,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:07,273 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:07,806 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:08,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:08,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:09,807 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:11,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:11,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:11,808 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:13,179 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:13,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:13,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:13,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:15,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:16,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:16,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:16,702 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:56:16,703 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:56:16,703 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:56:16,704 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:56:16,810 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:56:17,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:18,346 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:18,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:18,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:18,813 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:19,813 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:19,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:19,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:19,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:20,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:20,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:21,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:23,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:23,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:23,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:24,246 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:25,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:25,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:25,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:27,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:29,342 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:56:29,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:29,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:29,414 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:29,817 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:31,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:32,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:32,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:33,818 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:34,292 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:56:34,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:34,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:34,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:34,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:34,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:35,088 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:35,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:36,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:36,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:37,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:39,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:39,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:39,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:40,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:41,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:41,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:41,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:43,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:44,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:44,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:45,406 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:45,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:46,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:46,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:47,824 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:49,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:49,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:50,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:50,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:51,308 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:51,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:52,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:52,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:53,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:55,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:55,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:55,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:56,527 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:56,540 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:56:56,542 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:56:56,543 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:56:56,543 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:56:56,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:56:57,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:57,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:57,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:58,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:56:59,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:59,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:59,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:01,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:02,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:02,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:02,391 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:03,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:04,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:04,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:04,293 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:57:04,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:04,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:04,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:05,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:07,738 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:07,829 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:57:07,832 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:07,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:07,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:09,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:10,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:10,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:11,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:12,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:13,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:13,489 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:13,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:15,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:15,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:15,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:17,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:18,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:18,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:19,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:19,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:19,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:19,862 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:19,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:20,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:20,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:21,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:22,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:22,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:23,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:24,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:24,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:24,534 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:25,839 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:26,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:26,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:27,839 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:29,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:29,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:29,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:30,079 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:31,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:31,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:31,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:33,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:33,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:33,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:34,296 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:57:34,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:34,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:34,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:35,089 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:35,842 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:36,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:57:36,163 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:57:36,163 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:57:36,164 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:57:36,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:36,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:36,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:57:37,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:38,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:38,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:38,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:39,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:40,332 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:40,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:40,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:41,845 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:43,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:44,241 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:57:44,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:44,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:45,847 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:45,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:47,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:47,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:47,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:49,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:49,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:49,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:49,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:50,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:51,849 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:52,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:52,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:53,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:54,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:54,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:55,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:56,352 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:57,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:57,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:57,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:57:59,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:59,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:59,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:01,456 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:01,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:01,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:01,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:03,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:04,297 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:58:04,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:04,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:04,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:04,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:04,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:05,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:06,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:06,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:07,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:07,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:09,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:09,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:09,856 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:12,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:12,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:12,975 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:14,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:14,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:16,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:16,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:17,976 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:19,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:19,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:19,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:19,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:21,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:23,038 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:58:23,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:23,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:23,696 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:23,861 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:25,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:25,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:25,861 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:27,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:27,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:27,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:28,398 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:58:28,399 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:58:28,400 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:58:28,400 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:58:28,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:58:29,401 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:29,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:30,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:30,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:30,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:31,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:32,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:32,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:33,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:34,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:34,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:34,300 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:58:34,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:34,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:35,082 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:35,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:37,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:37,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:37,867 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:39,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:39,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:39,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:40,471 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:41,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:41,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:41,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:43,869 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:44,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:44,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:45,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:46,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:46,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:46,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:47,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:48,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:48,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:49,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:49,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:49,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:51,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:51,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:51,526 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:51,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:53,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:53,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:53,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:55,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:55,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:55,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:57,108 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:57,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:58:59,112 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:58:59,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:59,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:59,875 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:01,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:02,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:02,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:03,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:03,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:04,300 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:59:04,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:04,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:04,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:04,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:04,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:05,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:06,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:06,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:07,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:08,243 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:08,256 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:59:08,258 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:59:08,258 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:59:08,259 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:59:08,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:59:09,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:09,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:09,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:10,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:11,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:11,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:11,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:13,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:13,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:13,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:14,160 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:15,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:16,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:16,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:17,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:18,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:18,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:19,319 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:19,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:19,863 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:19,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:19,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:20,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:20,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:21,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:22,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:22,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:23,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:25,232 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:25,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:25,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:25,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:27,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:27,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:27,886 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:29,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:29,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:29,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:30,349 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:31,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:32,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:32,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:33,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:34,302 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 15:59:34,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:34,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:35,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:36,283 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:36,422 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:59:36,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:36,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:37,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:38,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:38,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:39,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:41,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:41,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:41,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:41,891 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:43,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:43,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:43,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:45,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:45,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:45,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:47,275 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:47,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:47,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:59:47,924 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 15:59:47,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:59:47,924 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:59:48,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:48,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:48,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 15:59:49,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:49,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:49,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:49,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:50,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:50,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:50,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:51,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:52,828 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:52,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:52,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:53,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:55,896 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:56,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:56,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:57,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 15:59:58,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:58,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:58,375 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:59,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:00,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:00,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:01,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:03,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:03,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:03,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:03,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:04,304 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:00:04,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:04,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:04,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:05,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:05,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:05,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:07,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:07,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:07,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:09,670 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:09,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:10,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:10,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:11,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:13,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:14,181 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:00:14,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:14,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:14,981 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:15,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:17,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:17,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:17,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:19,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:19,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:19,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:19,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:19,904 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:20,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:21,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:22,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:22,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:23,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:24,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:24,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:25,474 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:25,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:26,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:26,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:27,756 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:00:27,757 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:00:27,758 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:00:27,758 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:00:27,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:27,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:00:29,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:29,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:29,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:30,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:31,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:31,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:31,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:31,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:33,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:34,305 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:00:34,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:34,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:34,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:34,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:35,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:36,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:36,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:36,532 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:37,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:38,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:38,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:39,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:41,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:41,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:41,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:42,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:43,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:43,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:43,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:45,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:46,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:46,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:47,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:48,033 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:48,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:48,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:49,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:49,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:49,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:51,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:52,562 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:00:53,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:53,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:53,195 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:53,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:55,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:55,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:55,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:57,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:00:58,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:58,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:59,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:59,920 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:00,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:00,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:01,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:02,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:02,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:03,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:04,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:04,308 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:01:04,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:04,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:04,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:05,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:05,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:05,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:07,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:07,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:07,536 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:01:07,538 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:01:07,538 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:01:07,538 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:01:07,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:07,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:01:08,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:09,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:09,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:09,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:10,160 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:12,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:12,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:12,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:14,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:14,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:14,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:15,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:16,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:16,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:16,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:18,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:19,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:19,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:19,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:20,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:21,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:21,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:21,281 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:22,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:24,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:24,141 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:24,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:26,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:26,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:26,836 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:26,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:28,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:30,297 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:01:30,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:30,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:30,936 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:32,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:32,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:32,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:32,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:34,310 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:01:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:34,864 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:34,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:35,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:35,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:36,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:37,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:37,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:37,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:38,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:39,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:39,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:40,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:42,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:42,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:42,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:43,614 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:44,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:44,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:46,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:46,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:48,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:48,986 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:49,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:49,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:49,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:50,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:51,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:51,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:52,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:01:52,906 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:01:52,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:01:52,906 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:01:52,944 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:52,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:01:53,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:53,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:54,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:54,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:56,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:56,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:56,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:58,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:01:58,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:58,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:00,279 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:00,947 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:00,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:00,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:02,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:04,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:04,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:04,311 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:02:04,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:04,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:04,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:04,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:05,499 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:06,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:07,755 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:02:07,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:07,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:08,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:09,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:09,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:10,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:11,392 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:12,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:12,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:12,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:14,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:14,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:15,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:16,497 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:16,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:17,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:17,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:18,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:19,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:19,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:20,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:20,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:20,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:22,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:22,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:22,409 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:22,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:24,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:24,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:25,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:26,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:27,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:27,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:27,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:28,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:29,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:29,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:30,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:32,252 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:02:32,253 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:02:32,254 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:02:32,254 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:02:32,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:32,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:32,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:32,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:02:33,255 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:34,312 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:02:34,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:34,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:34,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:34,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:34,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:34,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:36,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:36,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:36,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:38,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:38,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:39,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:39,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:40,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:41,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:41,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:42,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:44,469 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:44,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:45,507 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:02:45,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:45,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:46,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:48,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:48,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:48,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:49,526 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:49,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:49,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:50,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:50,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:50,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:52,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:52,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:53,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:54,858 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:54,968 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:55,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:55,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:56,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:02:57,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:57,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:58,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:00,678 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:00,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:00,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:01,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:02,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:03,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:03,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:04,314 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:03:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:04,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:04,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:05,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:05,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:06,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:06,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:08,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:08,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:08,973 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:10,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:10,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:10,974 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:11,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:12,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:12,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:12,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:03:12,661 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:03:12,661 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:03:12,662 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:03:12,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:12,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:03:14,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:15,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:15,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:16,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:17,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:17,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:17,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:18,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:19,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:19,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:19,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:19,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:19,865 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:20,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:22,324 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:22,860 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:03:22,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:24,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:24,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:24,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:26,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:26,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:26,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:27,361 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:28,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:28,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:28,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:30,981 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:31,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:31,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:32,423 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:32,982 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:33,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:33,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:34,315 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:03:34,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:34,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:34,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:34,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:35,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:35,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:36,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:38,074 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:38,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:38,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:38,984 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:40,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:40,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:40,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:42,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:42,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:42,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:43,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:44,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:45,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:45,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:46,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:47,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:47,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:48,703 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:48,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:49,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:49,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:49,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:49,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:49,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:50,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:52,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:52,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:52,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:53,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:53,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:03:53,901 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:03:53,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:03:53,902 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:03:53,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:03:54,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:54,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:54,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:56,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:56,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:56,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:58,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:03:59,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:00,183 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:04:00,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:00,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:01,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:02,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:03,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:03,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:04,317 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:04:04,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:04,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:04,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:04,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:04,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:06,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:06,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:06,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:08,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:08,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:08,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:10,098 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:10,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:10,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:10,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:12,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:13,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:13,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:14,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:15,291 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:15,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:15,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:16,998 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:17,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:17,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:18,998 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:19,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:19,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:19,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:20,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:20,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:20,999 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:21,241 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:22,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:22,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:23,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:24,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:24,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:26,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:27,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:27,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:29,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:29,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,801 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:34,319 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:04:34,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:34,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:34,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:37,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:37,292 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:38,579 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:04:38,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:38,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:39,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:40,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:40,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:41,006 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:42,721 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:43,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:43,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:43,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:45,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:45,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:45,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:46,639 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:04:46,641 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:04:46,641 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:04:46,642 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:04:47,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:47,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:04:47,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:47,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:48,267 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:49,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:49,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:49,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:49,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:50,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:50,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:51,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:52,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:52,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:53,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:53,779 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:55,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:55,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:55,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:57,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:57,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:57,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:59,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:04:59,329 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:59,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:59,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:01,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:03,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:03,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:03,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:04,320 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:05:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:04,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:05,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:05,088 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:05,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:05,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:07,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:07,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:07,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:09,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:10,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:10,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:10,289 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:11,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:12,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:12,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:13,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:15,019 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:16,002 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:05:16,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:16,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:16,204 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:17,020 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:19,021 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:19,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:19,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:19,865 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:19,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:19,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:21,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:21,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:21,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:21,387 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:23,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:23,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:23,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:25,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:26,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:26,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:26,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:05:26,327 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:05:26,327 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:05:26,327 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:05:27,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:27,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:05:27,328 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:28,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:28,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:29,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:30,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:30,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:31,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:32,591 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:33,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:33,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:33,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:34,322 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:05:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:34,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:35,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:35,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:35,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:37,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:37,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:37,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:38,505 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:39,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:40,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:40,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:41,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:42,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:42,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:43,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:43,641 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:45,032 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:45,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:45,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:47,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:47,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:47,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:49,034 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:49,600 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:49,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:49,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:49,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:49,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:49,866 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:51,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:53,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:53,333 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:05:53,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:53,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:55,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:55,154 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:56,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:56,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:57,037 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:05:58,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:58,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:59,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:00,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:00,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:00,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:01,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:03,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:03,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:03,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:04,323 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:06:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:04,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:05,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:05,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:05,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:06,060 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:06,297 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:06:06,298 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:06:06,298 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:06:06,299 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:06:07,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:07,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:06:07,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:07,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:09,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:09,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:09,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:11,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:11,815 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:12,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:12,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:13,043 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:14,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:14,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:15,043 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:16,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:16,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:17,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:17,376 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:18,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:18,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:19,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:19,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:19,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:19,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:21,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:21,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:21,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:23,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:23,190 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:23,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:23,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:25,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:25,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:25,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:27,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:28,507 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:29,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:29,344 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:06:29,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:29,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:31,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:32,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:32,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:33,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:33,583 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:34,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:34,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:34,325 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:06:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:34,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:35,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:37,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:37,417 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:37,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:39,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:39,311 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:39,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:39,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:41,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:41,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:41,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:43,054 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:44,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:44,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:44,517 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:45,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:46,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:46,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:46,945 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:06:46,947 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:06:46,947 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:06:46,948 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:06:47,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:47,059 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:06:49,059 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:49,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:49,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:49,642 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:49,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:49,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:49,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:51,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:51,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:51,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:53,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:54,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:54,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:55,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:55,192 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:56,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:56,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:57,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:06:58,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:58,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:59,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:00,342 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:01,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:01,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:01,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:03,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:03,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:03,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:04,326 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:07:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:04,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:04,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:05,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:06,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:06,998 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:07:07,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:07,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:07,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:09,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:09,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:09,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:11,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:11,317 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:11,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:11,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:13,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:14,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:14,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:15,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:16,445 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:16,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:16,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:17,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:19,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:19,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:19,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:19,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:20,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:20,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:21,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:22,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:22,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:22,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:23,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:25,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:25,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:25,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:27,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:27,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:27,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:27,153 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:28,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:07:28,898 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:07:28,898 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:07:28,899 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:07:29,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:29,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:07:29,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:29,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:31,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:32,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:32,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:32,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:33,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:34,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:34,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:34,328 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:07:34,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:34,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:34,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:35,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:37,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:37,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:37,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:38,093 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:39,078 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:39,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:39,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:41,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:42,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:42,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:43,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:43,209 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:45,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:45,751 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:07:45,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:45,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:47,083 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:47,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:47,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:48,591 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:49,084 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:49,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:49,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:49,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:50,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:50,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:51,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:52,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:52,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:53,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:54,577 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:55,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:55,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:55,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:57,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:57,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:57,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:59,088 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:07:59,978 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:00,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:00,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:01,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:02,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:02,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:03,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:04,329 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:08:04,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:04,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:04,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:04,867 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:04,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:05,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:05,090 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:07,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:07,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:07,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:09,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:09,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:10,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:11,827 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:08:11,828 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:08:11,828 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:08:11,830 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:08:11,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:11,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:12,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:08:13,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:14,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:14,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:15,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:16,204 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:16,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:16,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:17,094 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:18,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:18,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:19,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:19,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:20,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:20,056 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:21,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:21,546 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:22,360 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:08:23,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:23,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:24,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:26,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:27,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:27,373 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:27,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:29,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:29,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:30,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:32,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:32,453 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:32,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:32,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:34,100 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:34,331 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:08:34,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:34,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:34,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:34,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:36,101 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:37,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:37,784 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:37,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:38,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:39,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:41,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:42,103 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:43,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:43,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:43,416 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:46,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:46,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:46,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:48,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:48,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:48,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:49,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:49,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:49,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:49,868 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:51,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:51,281 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:08:51,283 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:08:51,283 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:08:51,284 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:08:51,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:51,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:52,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:08:53,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:53,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:53,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:53,944 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:55,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:55,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:55,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:57,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:58,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:58,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:59,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:08:59,314 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:00,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:00,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:03,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:04,332 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:09:04,333 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:04,459 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:09:04,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:04,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:04,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:04,868 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:04,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:05,112 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:07,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:07,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:07,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:09,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:09,488 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:09,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:09,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:11,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:11,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:11,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:13,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:14,532 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:14,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:14,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:15,116 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:16,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:16,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:17,117 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:18,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:18,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:19,118 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:19,596 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:19,867 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:19,868 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:19,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:21,118 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:22,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:22,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:23,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:24,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:24,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:24,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:25,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:26,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:26,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:27,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:29,121 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:29,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:29,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:29,701 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:31,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:31,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:31,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:33,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:33,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:33,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:34,334 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:09:34,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:34,868 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:35,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:35,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:35,457 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:09:35,458 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:09:35,459 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:09:35,459 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:09:36,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:09:36,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:36,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:37,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:38,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:38,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:39,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:40,486 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:41,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:43,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:43,260 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:09:43,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:43,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:45,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:45,542 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:45,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:45,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:47,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:47,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:47,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:49,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:49,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:49,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:49,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:50,574 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:50,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:50,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:51,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:52,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:52,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:53,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:54,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:54,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:55,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:55,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:57,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:09:58,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:58,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:59,131 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:00,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:00,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:00,696 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:01,132 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:02,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:02,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:03,132 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:04,335 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:10:04,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:04,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:04,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:05,133 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:05,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:05,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:05,744 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:07,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:07,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:07,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:09,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:09,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:09,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:10,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:11,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:12,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:12,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:13,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:14,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:14,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:15,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:15,960 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:16,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:16,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:17,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:17,695 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:10:17,697 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:10:17,697 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:10:17,698 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:10:18,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:10:19,138 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:19,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:19,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:19,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:20,613 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:10:20,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:20,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:21,127 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:21,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:23,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:23,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:23,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:25,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:25,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:25,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:26,289 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:27,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:27,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:27,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:29,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:30,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:30,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:31,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:31,405 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:32,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:32,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:33,143 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:34,337 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:10:34,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:34,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:34,868 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:34,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:35,144 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:36,561 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:37,145 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:37,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:37,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:39,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:39,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:40,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:41,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:41,641 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:42,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:42,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:43,147 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:45,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:45,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:46,642 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:47,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:47,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:49,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:49,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:49,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:49,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:49,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:51,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:51,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:51,998 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:54,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:54,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:57,152 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:10:57,551 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:58,355 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:10:58,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:58,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:59,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:00,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:00,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:01,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:03,045 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:03,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:03,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:03,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:04,339 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:11:04,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:04,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:04,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:05,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:05,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:05,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:07,156 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:07,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:07,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:08,188 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:09,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:10,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:10,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:11,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:12,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:11:12,391 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:11:12,391 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:11:12,391 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:11:12,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:12,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:13,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:13,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:11:13,392 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:14,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:14,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:15,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:17,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:18,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:18,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:18,509 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:19,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:19,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:19,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:20,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:20,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:21,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:23,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:23,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:23,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:23,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:25,163 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:25,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:25,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:27,164 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:27,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:27,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:28,843 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:29,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:30,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:30,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:31,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:32,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:32,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:33,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:34,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:34,340 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:11:34,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:34,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:34,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:35,167 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:36,618 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:11:36,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:36,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:37,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:39,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:39,341 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:40,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:40,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:41,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:42,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:42,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:43,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:44,640 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:45,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:45,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:45,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:47,172 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:47,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:47,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:49,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:49,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:49,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:49,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:49,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:49,870 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:50,058 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:51,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:52,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:52,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:53,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:54,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:54,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:55,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:55,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:56,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:11:56,076 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:11:56,077 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:11:56,077 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:11:56,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:11:56,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:56,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:57,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:59,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:11:59,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:59,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:00,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:01,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:01,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:01,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:03,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:03,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:03,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:04,342 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:12:04,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:04,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:04,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:05,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:05,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:05,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:06,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:07,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:09,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:09,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:09,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:11,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:11,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:11,181 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:11,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:13,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:14,739 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:12:14,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:14,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:15,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:16,567 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:17,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:17,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:17,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:19,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:19,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:19,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:19,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:19,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:19,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:21,626 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:21,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:21,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:23,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:24,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:25,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:25,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:26,691 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:27,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:27,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:27,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:30,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:30,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:30,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:31,745 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:32,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:32,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:32,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:34,195 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:34,348 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:12:34,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:34,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:34,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:34,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:34,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:36,195 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:36,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:37,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:37,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:38,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:38,508 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:12:38,509 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:12:38,510 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:12:38,510 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:12:39,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:12:39,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:39,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:40,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:41,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:41,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:41,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:41,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:42,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:44,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:44,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:44,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:46,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:46,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:46,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:47,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:48,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:49,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:49,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:49,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:50,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:50,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:50,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:52,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:52,209 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:54,126 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:12:54,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:54,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:54,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:56,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:56,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:56,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:58,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:58,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:12:59,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:59,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:00,204 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:01,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:01,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:02,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:03,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:03,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:03,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:04,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:04,345 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:13:04,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:04,870 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:04,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:06,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:06,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:06,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:08,183 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:08,207 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:08,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:08,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:10,208 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:10,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:10,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:12,208 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:13,234 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:13,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:13,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:14,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:15,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:15,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:16,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:17,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:17,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:18,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:18,292 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:19,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:19,870 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:19,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:20,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:20,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:20,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:20,688 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:13:20,689 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:13:20,690 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:13:20,690 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:13:21,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:13:22,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:22,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:22,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:23,346 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:24,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:24,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:24,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:26,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:27,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:27,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:28,214 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:28,412 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:30,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:31,263 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:13:31,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:31,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:32,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:33,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:34,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:34,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:34,350 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:13:34,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:34,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:36,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:36,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:36,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:38,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:38,538 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:39,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:39,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:40,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:41,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:41,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:42,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:43,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:44,223 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:44,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:44,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:46,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:46,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:46,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:48,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:48,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:49,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:49,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:49,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:49,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:49,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:50,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:51,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:51,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:52,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:53,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:53,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:54,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:54,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:56,228 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:56,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:56,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:58,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:13:58,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:58,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:59,468 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:00,230 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:00,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:00,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:02,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:03,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:14:03,805 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:14:03,805 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:14:03,806 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:14:03,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:03,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:04,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:04,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:14:04,353 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:14:04,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:04,870 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:04,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:05,028 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:05,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:05,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:06,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:08,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:09,544 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:14:09,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:09,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:10,124 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:10,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:12,234 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:12,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:12,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:14,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:14,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:14,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:15,451 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:16,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:16,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:17,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:18,236 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:19,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:19,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:19,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:19,870 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:19,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:20,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:21,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:21,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:21,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:22,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:24,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:24,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:24,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:26,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:26,748 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:26,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:26,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:28,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:30,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:30,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:30,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:32,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:32,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:32,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:32,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:34,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:34,354 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:14:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:34,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:34,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:35,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:35,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:36,245 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:37,315 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:37,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:37,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:38,246 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:39,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:39,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:40,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:42,248 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:42,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:42,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:42,644 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:44,248 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:44,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:44,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:46,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:47,967 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:47,980 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:14:47,982 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:14:47,982 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:14:47,983 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:14:48,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:48,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:14:48,268 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:14:48,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:48,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:49,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:49,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:49,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:50,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:50,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:50,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:52,251 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:53,286 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:53,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:53,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:54,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:55,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:55,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:56,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:57,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:57,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:58,254 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:14:58,584 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:59,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:59,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:00,254 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:02,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:02,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:02,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:03,716 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:04,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:04,355 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:15:04,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:04,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:04,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:04,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:04,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:06,256 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:06,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:06,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:08,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:08,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:08,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:09,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:10,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:11,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:11,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:12,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:13,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:13,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:14,259 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:14,596 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:15,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:15,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:16,260 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:18,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:19,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:19,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:19,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:19,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:19,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:20,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:21,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:21,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:21,811 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:22,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:24,263 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:24,776 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:15:24,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:24,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:26,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:27,245 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:27,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:27,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:28,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:29,953 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:29,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:30,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:31,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:32,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:32,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:32,645 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:32,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:15:32,660 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:15:32,661 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:15:32,662 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:15:33,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:15:34,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:34,359 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:15:34,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:34,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:34,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:35,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:35,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:36,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:37,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:37,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:38,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:38,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:39,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:39,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:40,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:42,270 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:42,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:42,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:43,293 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:44,270 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:44,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:44,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:46,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:46,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:46,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:48,272 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:48,378 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:48,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:48,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:49,870 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:49,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:49,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:50,273 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:51,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:51,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:52,273 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:53,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:53,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:54,270 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:54,274 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:55,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:55,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:56,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:57,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:57,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:58,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:15:59,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:00,276 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:01,555 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:16:01,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:01,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:02,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:04,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:04,359 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:16:04,360 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:04,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:04,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:04,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:04,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:04,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:06,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:06,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:06,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:08,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:08,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:08,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:09,452 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:10,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:11,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:11,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:12,281 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:13,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:13,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:14,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:14,525 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:15,212 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:16:15,214 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:16:15,214 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:16:15,214 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:16:15,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:16:15,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:15,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:16,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:18,283 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:18,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:18,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:19,587 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:19,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:19,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:20,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:20,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:20,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:22,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:23,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:23,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:24,285 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:24,641 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:26,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:26,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:26,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:28,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:28,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:28,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:29,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:30,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:30,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:30,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:32,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:32,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:32,288 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:34,289 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:34,360 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:16:34,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:34,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:34,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:35,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:35,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:35,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:36,289 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:38,290 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:38,935 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:16:38,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:38,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:40,291 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:40,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:40,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:41,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:42,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:43,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:43,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:44,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:45,564 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:46,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:46,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:46,293 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:48,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:48,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:48,294 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:49,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:49,871 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:50,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:50,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:50,294 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:50,637 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:52,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:52,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:52,295 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:54,296 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:55,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:55,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:55,686 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:56,297 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:57,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:57,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:57,376 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:16:57,378 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:16:57,379 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:16:57,379 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:16:58,297 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:16:58,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:16:59,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:59,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:00,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:00,758 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:02,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:02,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:02,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:04,300 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:04,363 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:17:04,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:04,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:04,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:04,872 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:04,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:05,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:06,301 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:06,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:06,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:08,301 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:09,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:09,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:10,302 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:10,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:11,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:11,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:12,302 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:14,303 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:15,400 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:17:15,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:15,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:16,304 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:16,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:17,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:17,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:19,872 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:20,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:20,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:22,066 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:22,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:22,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:24,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:24,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:27,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:27,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:27,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:29,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:29,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:30,308 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:31,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:31,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:32,238 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:32,309 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:34,363 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:17:34,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:34,872 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:34,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:34,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:35,310 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:36,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:36,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:37,310 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:37,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:38,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:38,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:39,311 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:41,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:41,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:41,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:42,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:43,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:44,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:44,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:45,313 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:46,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:46,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:47,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:47,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:48,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:48,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:49,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:49,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:49,872 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:51,315 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:51,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:17:51,328 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:17:51,328 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:17:51,329 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:17:51,744 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:17:51,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:51,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:52,315 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:17:53,001 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:53,316 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:54,316 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:54,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:54,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:55,316 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:56,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:56,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:57,317 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:17:58,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:58,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:58,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:59,318 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:01,318 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:01,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:01,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:03,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:03,319 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:03,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:03,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:04,365 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:18:04,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:04,872 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:04,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:05,320 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:07,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:07,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:07,321 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:08,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:09,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:09,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:09,321 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:11,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:11,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:11,322 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:13,250 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:13,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:14,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:14,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:15,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:16,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:16,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:17,324 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:18,300 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:18,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:18,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:19,325 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:19,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:19,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:21,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:21,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:21,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:23,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:23,367 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:23,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:23,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:25,327 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:25,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:25,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:27,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:28,419 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:29,179 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:18:29,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:29,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:29,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:31,329 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:32,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:32,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:33,330 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:33,468 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:33,482 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:18:33,484 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:18:33,484 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:18:33,485 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:18:34,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:34,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:34,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:18:34,367 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:18:34,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:34,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:35,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:36,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:36,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:36,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:37,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:38,546 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:39,332 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:39,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:39,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:41,333 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:41,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:41,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:43,334 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:43,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:43,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:43,601 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:45,335 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:46,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:46,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:47,335 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:48,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:48,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:48,668 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:49,336 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:49,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:49,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:49,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:50,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:50,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:51,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:52,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:52,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:53,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:53,743 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:55,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:55,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:55,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:57,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:57,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:57,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:58,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:59,339 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:18:59,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:59,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:01,340 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:02,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:02,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:03,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:03,856 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:04,369 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:19:04,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:04,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:05,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:06,494 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:19:06,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:06,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:07,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:08,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:08,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:08,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:09,343 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:11,344 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:11,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:11,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:13,345 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:13,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:13,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:14,790 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:15,345 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:15,488 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:19:15,490 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:19:15,490 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:19:15,490 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:19:15,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:15,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:16,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:19:17,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:17,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:17,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:18,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:19,347 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:19,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:19,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:19,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:20,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:20,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:21,348 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:22,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:22,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:23,348 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:24,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:25,349 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:26,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:26,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:27,350 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:28,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:28,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:29,351 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:29,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:30,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:30,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:31,351 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:33,352 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:33,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:33,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:34,371 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:19:34,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:34,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:34,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:35,055 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:35,353 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:35,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:35,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:37,354 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:37,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:37,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:39,354 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:40,708 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:40,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:40,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:41,355 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:43,356 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:44,582 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:19:44,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:44,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:45,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:45,766 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:47,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:47,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:49,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:49,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:49,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:49,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:49,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:50,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:51,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:52,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:52,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:53,359 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:54,695 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:19:54,697 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:19:54,697 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:19:54,698 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:19:54,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:54,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:55,360 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:55,360 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:19:56,361 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:56,361 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:57,361 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:19:58,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:58,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:00,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:00,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:01,361 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:03,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:03,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:03,363 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:04,372 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:20:04,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:04,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:05,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:05,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:05,364 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:06,362 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:07,365 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:08,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:08,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:09,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:10,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:10,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:11,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:11,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:12,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:12,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:13,367 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:15,368 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:15,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:15,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:16,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:17,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:17,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:17,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:19,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:19,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:19,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:19,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:19,873 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:21,370 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:21,593 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:23,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:24,528 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:20:24,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:24,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:25,376 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:26,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:26,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:26,671 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:27,376 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:28,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:28,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:29,377 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:31,378 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:31,720 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:31,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:31,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:33,379 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:33,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:33,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:34,374 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:20:34,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:34,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:35,379 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:35,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:35,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:36,787 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:37,380 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:38,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:38,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:39,381 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:40,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:20:40,868 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:20:40,869 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:20:40,869 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:20:41,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:41,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:41,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:41,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:20:41,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:42,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:43,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:43,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:43,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:46,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:46,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:46,872 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:48,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:48,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:49,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:49,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:50,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:50,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:52,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:53,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:53,408 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:55,386 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:20:55,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:55,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:57,647 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:58,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:58,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:02,287 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:21:02,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:02,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:02,648 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:04,376 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:21:04,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:04,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:04,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:05,408 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:05,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:07,390 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:21:07,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:07,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:08,441 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:10,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:10,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:12,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:12,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:13,442 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:15,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:15,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:17,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:17,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:18,443 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:19,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:19,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:20,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:20,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:22,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:22,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:23,396 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:21:23,753 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:25,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:25,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:27,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:27,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:28,754 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:30,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:30,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:32,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:33,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:33,755 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:34,378 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:21:34,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:34,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:35,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:35,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:35,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:21:38,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:38,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:39,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:41,797 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:21:41,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:41,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:43,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:44,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:45,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:47,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:47,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:49,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:49,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:49,404 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:21:49,660 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:49,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:49,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:49,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:52,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:52,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:54,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:54,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:55,227 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:56,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:56,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:59,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:59,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:00,231 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:01,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:01,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:03,408 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:22:03,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:03,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:04,379 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:22:04,873 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:04,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:04,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:06,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:06,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:06,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:08,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:08,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:10,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:10,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:11,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:13,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:13,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:15,412 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:22:15,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:15,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:16,573 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:19,432 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:22:19,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:19,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:19,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:19,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:22,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:22,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:22,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:24,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:24,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:26,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:26,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:27,045 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:29,416 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:22:29,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:29,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:31,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:31,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:32,526 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:33,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:33,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:34,382 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:22:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:34,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:34,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:36,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:36,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:38,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:38,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:38,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:40,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:40,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:43,420 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:22:43,854 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:43,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:43,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:45,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:45,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:47,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:47,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:48,855 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:49,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:51,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:51,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:53,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:53,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:54,081 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:55,424 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:22:56,899 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:22:56,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:56,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:59,324 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:00,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:00,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:02,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:02,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:04,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:04,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:04,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:04,382 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:23:04,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:04,874 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:04,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:07,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:07,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:09,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:09,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:09,428 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:23:09,712 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:12,468 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:12,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:14,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:14,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:14,713 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:16,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:16,583 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:19,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:19,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:19,713 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:19,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:19,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:21,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:21,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:23,432 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:23:23,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:23,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:24,829 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:27,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:27,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:29,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:29,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:29,830 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:32,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:32,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:34,384 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:23:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:34,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:34,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:35,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:36,305 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:23:36,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:36,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:37,437 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:23:39,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:39,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:40,645 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:41,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:41,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:43,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:43,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:45,646 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:46,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:46,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:48,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:48,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:49,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:49,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:50,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:50,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:51,441 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:23:51,622 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:53,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:53,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:55,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:55,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:56,623 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:57,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:57,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:00,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:00,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:01,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:02,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:02,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:04,385 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:24:04,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:04,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:05,446 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:24:05,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:05,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:07,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:07,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:07,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:09,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:09,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:12,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:13,472 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:24:13,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:13,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:16,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:16,698 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:17,057 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:18,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:18,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:19,450 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:24:19,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:19,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:19,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:20,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:20,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:23,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:23,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:23,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:25,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:25,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:27,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:27,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:28,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:30,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:30,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:32,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:32,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:33,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:33,455 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:24:34,387 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:24:34,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:34,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:34,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:34,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:34,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:38,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:38,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:39,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:40,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:40,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:42,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:42,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:44,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:45,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:45,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:45,459 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:24:47,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:47,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:49,307 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:49,875 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:50,983 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:24:51,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:51,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:54,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:54,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:55,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:56,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:56,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:59,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:59,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:59,463 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:25:00,497 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:01,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:01,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:03,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:03,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:04,390 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:25:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:04,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:06,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:06,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:06,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:08,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:08,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:10,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:10,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:11,468 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:25:11,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:13,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:13,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:15,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:15,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:16,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:17,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:17,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:19,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:19,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:19,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:20,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:20,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:22,089 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:22,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:22,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:24,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:24,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:25,472 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:25:27,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:28,125 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:25:28,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:28,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:31,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:31,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:32,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:33,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:33,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:34,390 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:25:34,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:34,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:34,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:35,351 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:35,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:37,249 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:38,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:38,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:39,481 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:25:40,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:40,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:42,294 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:42,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:42,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:45,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:45,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:47,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:47,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:47,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:49,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:49,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:49,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:49,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:51,484 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:25:52,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:52,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:52,616 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:54,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:54,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:57,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:57,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:57,617 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:59,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:59,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:01,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:01,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:02,618 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:04,391 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:26:04,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:04,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:04,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:05,459 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:26:05,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:05,488 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:26:05,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:08,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:08,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:08,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:10,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:10,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:12,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:12,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:13,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:15,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:15,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:17,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:17,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:18,071 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:19,493 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:26:19,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:19,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:19,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:19,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:19,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:22,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:22,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:24,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:24,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:24,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:26,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:26,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:29,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:30,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:30,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:31,496 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:26:32,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:32,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:34,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:34,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:34,393 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:26:34,395 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:34,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:34,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:34,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:37,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:37,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:39,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:39,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:40,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:43,035 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:26:43,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:43,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:45,501 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:26:45,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:46,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:46,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:48,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:48,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:49,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:49,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:49,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:50,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:51,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:51,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:53,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:53,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:55,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:56,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:56,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:58,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:58,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:59,506 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:27:01,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:01,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:01,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:03,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:03,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:04,395 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:27:04,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:04,876 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:04,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:06,054 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:06,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:06,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:08,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:08,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:11,253 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:11,510 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:27:12,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:12,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:14,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:14,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:16,254 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:17,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:17,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:19,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:19,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:19,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:19,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:19,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:21,517 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:23,108 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:27:23,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:24,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:24,514 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:27:26,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:26,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:27,448 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:28,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:28,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:31,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:31,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:32,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:33,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:33,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:34,397 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:27:34,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:34,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:34,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:36,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:36,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:37,766 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:38,519 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:27:38,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:38,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:40,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:40,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:42,767 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:43,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:43,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:45,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:45,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:47,768 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:48,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:48,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:49,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:49,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:49,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:50,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:50,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:52,526 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:27:52,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:52,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:53,095 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:55,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:55,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:57,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:57,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:58,095 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:01,547 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:28:01,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:01,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:03,096 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:04,398 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:28:04,530 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:28:04,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:04,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:04,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:04,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:04,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:06,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:06,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:09,083 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:09,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:09,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:11,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:11,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:13,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:13,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:14,084 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:16,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:16,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:18,536 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:28:18,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:18,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:19,426 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:19,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:19,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:19,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:21,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:21,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:23,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:23,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:25,240 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:25,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:25,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:28,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:28,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:30,445 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:30,539 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:28:30,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:31,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:33,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:33,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:34,399 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:28:34,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:34,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:34,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:36,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:36,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:36,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:40,002 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:28:40,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:40,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:41,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:42,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:42,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:44,544 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:28:45,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:45,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:46,314 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:47,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:47,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:49,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:49,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:49,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:49,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:49,877 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:51,762 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:52,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:52,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:54,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:54,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:56,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:56,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:56,548 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:28:57,432 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:59,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:59,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:01,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:01,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:02,433 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:03,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:03,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:04,498 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:29:04,876 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:05,146 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:05,147 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:06,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:06,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:08,317 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:08,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:08,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:10,553 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:29:10,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:10,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:12,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:12,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:13,697 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:16,192 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:29:16,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:16,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:18,698 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:19,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:19,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:19,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:19,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:19,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:21,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:21,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:23,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:23,941 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:24,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:24,557 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:29:26,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:26,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:28,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:28,698 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:28,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:31,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:31,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:33,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:33,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:33,943 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:34,403 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:29:34,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:34,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:35,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:35,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:36,561 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:29:39,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:39,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:39,336 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:41,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:41,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:43,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:43,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:44,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:46,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:46,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:48,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:48,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:49,517 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:49,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:49,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:49,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:50,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:50,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:50,566 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:29:53,843 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:29:53,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:53,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:55,094 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:56,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:56,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:58,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:58,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:00,095 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:00,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:02,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:02,570 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:30:04,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:04,493 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:30:04,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:04,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:04,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:04,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:06,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:06,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:06,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:09,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:09,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:11,039 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:11,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:11,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:14,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:14,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:16,574 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:30:16,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:16,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:16,945 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:18,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:18,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:19,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:19,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:19,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:21,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:21,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:22,190 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:23,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:23,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:25,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:25,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:27,192 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:29,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:29,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:30,578 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:30:32,217 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:32,873 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:30:32,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:32,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:34,406 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:30:34,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:34,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:34,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:34,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:38,020 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:38,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:38,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:40,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:40,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:42,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:42,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:42,582 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:30:43,444 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:45,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:45,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:47,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:47,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:48,445 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:49,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:49,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:49,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:49,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:52,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:52,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:54,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:54,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:54,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:56,586 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:30:56,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:56,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:59,662 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:59,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:59,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:01,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:01,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:03,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:03,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:04,407 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:31:04,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:04,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:04,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:05,066 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:06,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:06,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:10,108 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:10,589 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:31:10,633 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:31:10,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:10,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:13,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:13,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:15,109 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:15,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:15,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:17,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:18,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:19,877 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:19,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:19,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:21,066 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:21,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:21,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:22,593 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:31:23,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:23,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:25,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:25,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:26,611 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:27,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:27,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:30,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:30,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:31,611 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:32,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:32,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:34,409 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:31:34,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:34,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:34,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:36,596 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:31:37,093 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:37,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:37,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:39,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:39,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:41,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:41,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:42,093 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:43,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:43,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:47,094 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:47,141 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:31:47,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:47,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:49,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:49,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:50,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:50,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:50,601 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:31:52,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:52,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:53,014 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:54,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:54,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:57,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:57,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:58,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:59,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:59,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:01,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:01,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:02,607 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:32:03,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:04,411 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:32:04,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:04,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:04,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:04,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:04,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:06,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:06,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:09,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:09,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:09,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:11,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:11,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:14,068 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:15,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:15,018 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:16,611 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:32:17,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:17,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:19,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:19,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:19,878 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:19,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:20,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:20,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:22,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:22,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:24,794 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:26,121 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:32:26,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:26,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:28,615 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:32:29,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:29,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:30,790 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:31,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:31,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:33,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:33,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:34,414 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:32:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:34,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:34,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:36,084 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:36,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:36,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:38,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:38,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:40,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:40,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:41,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:42,619 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:32:43,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:43,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:45,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:45,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:46,213 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:47,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:47,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:49,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:49,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:49,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:50,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:50,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:52,115 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:52,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:52,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:54,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:54,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:56,625 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:32:57,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:57,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:57,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:00,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:00,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:02,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:03,666 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:33:03,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:03,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:04,415 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:33:04,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:04,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:04,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:06,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:06,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:07,572 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:08,627 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:33:08,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:08,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:10,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:10,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:12,615 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:13,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:13,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:15,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:15,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:17,615 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:17,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:17,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:19,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:19,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:21,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:21,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:22,633 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:33:22,676 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:23,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:23,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:25,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:25,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:27,677 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:28,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:28,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:30,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:30,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:32,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:32,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:32,677 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:34,416 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:33:34,637 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:33:34,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:34,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:34,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:35,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:35,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:37,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:37,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:38,030 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:40,979 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:33:40,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:41,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:43,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:44,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:44,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:46,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:46,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:48,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:48,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:48,132 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:48,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:33:49,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:49,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:49,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:51,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:51,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:53,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:53,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:54,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:55,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:55,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:58,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:58,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:59,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:00,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:00,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:02,650 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:34:03,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:03,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:04,419 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:34:04,421 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:04,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:04,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:04,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:05,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:05,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:08,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:08,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:10,045 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:10,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:10,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:12,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:12,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:14,654 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:34:15,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:15,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:15,638 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:19,265 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:34:19,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:19,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:19,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:19,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:21,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:21,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:21,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:24,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:24,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:26,051 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:26,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:26,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:28,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:34:29,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:29,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:31,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:31,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:31,850 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:34,420 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:34:34,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:34,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:34,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:34,879 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:34,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:36,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:36,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:37,087 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:38,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:38,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:40,664 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:34:41,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:41,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:42,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:44,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:44,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:49,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:49,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:49,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:52,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:52,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:53,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:54,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:54,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:54,669 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:34:57,795 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:34:57,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:57,828 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:58,387 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:00,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:00,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:02,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:02,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:03,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:04,423 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:35:04,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:04,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:04,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:04,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:04,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:07,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:07,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:08,674 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:35:08,736 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:09,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:09,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:13,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:13,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:13,737 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:15,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:15,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:17,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:17,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:18,738 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:19,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:19,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:19,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:20,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:20,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:20,678 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:35:22,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:22,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:24,087 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:25,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:25,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:27,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:27,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:29,088 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:30,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:30,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:32,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:32,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:34,289 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:34,424 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:35:34,686 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:35:34,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:34,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:34,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:35,816 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:35:35,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:35,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:38,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:38,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:40,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:40,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:40,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:42,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:42,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:45,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:45,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:45,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:47,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:47,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:48,689 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:35:49,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:49,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:49,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:50,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:50,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:53,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:53,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:55,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:55,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:55,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:57,135 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:57,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:59,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:59,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:00,693 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:36:00,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:02,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:02,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:04,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:04,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:04,426 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:36:04,879 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:04,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:04,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:06,078 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:06,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:06,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:09,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:09,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:11,078 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:13,170 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:36:13,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:13,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:14,697 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:36:15,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:15,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:16,242 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:18,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:18,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:19,882 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:20,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:20,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:22,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:22,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:22,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:25,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:25,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:27,500 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:27,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:27,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:28,701 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:36:29,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:29,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:32,501 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:32,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:32,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:34,427 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:36:34,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:34,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:34,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:34,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:38,059 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:38,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:38,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:40,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:40,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:40,705 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:36:42,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:42,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:43,963 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:44,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:44,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:47,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:47,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:48,964 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:49,880 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:50,908 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:36:50,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:50,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:52,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:52,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:54,529 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:54,709 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:36:56,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:56,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:58,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:58,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:59,530 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:00,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:00,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:03,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:03,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:04,429 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:37:04,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:04,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:05,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:05,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:05,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:07,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:07,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:08,713 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:37:10,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:10,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:10,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:12,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:12,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:14,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:14,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:15,117 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:17,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:17,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:19,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:19,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:19,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:19,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:21,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:21,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:21,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:22,717 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:37:24,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:24,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:26,081 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:28,423 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:37:28,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:28,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:30,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:31,082 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:31,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:33,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:33,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:34,507 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:37:34,721 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:37:34,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:34,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:35,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:35,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:36,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:37,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:37,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:40,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:40,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:41,390 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:43,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:43,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:45,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:45,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:46,391 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:48,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:48,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:48,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:37:49,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:49,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:49,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:50,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:50,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:51,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:52,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:52,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:54,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:54,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:56,516 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:57,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:57,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:59,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:59,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:01,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:01,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:02,415 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:02,730 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:38:04,432 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:38:04,880 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:04,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:05,122 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:38:05,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:05,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:08,109 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:08,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:08,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:10,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:10,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:12,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:12,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:13,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:14,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:14,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:14,734 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:38:17,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:17,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:18,298 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:19,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:19,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:19,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:22,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:22,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:23,853 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:24,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:24,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:27,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:27,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:28,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:38:29,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:29,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:29,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:32,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:32,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:34,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:34,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:34,433 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:38:34,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:34,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:35,134 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:37,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:37,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:39,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:39,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:40,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:40,742 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:38:42,650 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:38:42,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:42,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:47,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:47,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:49,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:49,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:49,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:50,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:50,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:51,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:52,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:52,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:54,747 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:38:55,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:55,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:56,748 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:57,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:57,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:00,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:00,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:01,749 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:02,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:02,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:04,435 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:39:04,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:04,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:04,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:05,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,421 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:08,752 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:39:10,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:10,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:12,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:12,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:12,422 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:14,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:14,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:16,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:16,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:17,423 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:19,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:19,882 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:20,556 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:39:20,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:20,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:20,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:39:23,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:23,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:23,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:25,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:25,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:27,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:27,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:28,087 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:30,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:30,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:32,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:32,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:33,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:34,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:34,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:34,437 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:39:34,761 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:39:34,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:34,881 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:36,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:36,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:38,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:39,023 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:40,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:42,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:42,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:44,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:45,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:45,489 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:46,765 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:39:48,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:48,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:49,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:49,882 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:50,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:50,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:50,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:53,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:53,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:55,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:55,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:55,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:59,034 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:39:59,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:59,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:00,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:40:00,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:01,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:01,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:03,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:03,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:04,439 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:40:04,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:04,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:04,882 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:06,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:06,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:06,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:08,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:08,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:11,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:11,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:11,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:12,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:40:13,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:13,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:16,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:16,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:16,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:18,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:18,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:19,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:20,243 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:20,244 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:21,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:21,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:21,478 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:23,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:24,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:26,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:26,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:26,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:40:27,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:29,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:29,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:31,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:31,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:32,371 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:34,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:34,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:34,439 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:40:34,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:34,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:34,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:37,880 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:40:37,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:37,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:38,057 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:40,785 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:40:40,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:40,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:42,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:42,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:43,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:45,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:45,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:47,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:47,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:48,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:49,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:50,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:50,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:52,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:52,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:52,789 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:40:54,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:55,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:55,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:57,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:57,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:59,326 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:00,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:00,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:02,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:02,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:04,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:04,441 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:41:04,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:04,882 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:04,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:05,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:05,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:06,793 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:41:07,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:07,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:10,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:10,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:10,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:12,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:12,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:14,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:41:15,636 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:16,105 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:41:16,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:16,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:18,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:19,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:19,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:19,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:19,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:20,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:21,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:21,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:21,798 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:41:23,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:23,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:25,800 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:25,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:25,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:28,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:28,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:30,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:30,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:31,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:31,801 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:41:33,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:33,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:34,443 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:41:34,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:34,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:35,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:35,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:37,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:38,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:38,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:40,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:40,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:42,038 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:43,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:43,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:45,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:41:46,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:46,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:47,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:48,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:48,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:49,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:49,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:49,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:50,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:50,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:53,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:54,494 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:41:54,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:54,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:57,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:57,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:58,629 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:59,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:59,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:59,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:02,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:02,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:03,630 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:04,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:04,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:04,444 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:42:04,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:04,883 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:07,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:07,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:09,105 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:09,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:09,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:11,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:12,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:12,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:14,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:14,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:15,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:17,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:17,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:18,716 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:42:18,718 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:42:18,718 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:42:18,719 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:42:18,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:42:19,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:19,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:19,814 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:19,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:20,694 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:21,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:21,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:22,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:24,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:24,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:25,695 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:26,495 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:42:27,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:27,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:27,817 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:29,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:29,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:31,538 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:33,217 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:42:34,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:34,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:34,669 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:42:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:34,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:36,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:37,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:37,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:37,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:39,820 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:39,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:39,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:41,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:41,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:42,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:42,240 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:43,821 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:44,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:44,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:45,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:46,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:46,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:47,388 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:47,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:49,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:49,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:49,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:49,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:49,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:49,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:51,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:51,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:51,824 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:53,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:53,824 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:54,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:54,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:55,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:56,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:57,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:57,825 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:59,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:59,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:42:59,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:00,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:01,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:03,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:03,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:03,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:04,447 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:43:04,448 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:04,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:05,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:05,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:05,827 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:07,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:08,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:08,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:09,678 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:09,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:10,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:10,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:11,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:13,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:14,157 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:43:14,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:14,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:14,699 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:15,830 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:16,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:16,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:17,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:18,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:43:18,035 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:43:18,036 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:43:18,036 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:43:18,831 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:43:18,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:18,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:19,832 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:19,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:19,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:20,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:21,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:22,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:22,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:23,833 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:24,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:24,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:25,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:26,347 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:27,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:27,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:27,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:29,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:29,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:29,835 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:31,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:32,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:32,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:32,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:33,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:34,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:34,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:34,449 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:43:34,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:34,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:34,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:35,837 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:36,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:36,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:37,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:37,993 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:38,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:38,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:39,839 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:41,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:41,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:41,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:43,840 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:43,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:44,018 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:44,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:45,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:46,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:46,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:47,842 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:49,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:49,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:49,642 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:49,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:49,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:49,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:49,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:51,843 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:52,596 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:43:52,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:52,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:53,844 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:55,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:55,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:55,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:55,845 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:57,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:57,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:57,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:59,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:43:59,636 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:43:59,636 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:43:59,637 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:43:59,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:43:59,847 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:44:00,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:00,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:00,637 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:01,847 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:02,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:02,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:03,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:04,451 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:44:04,883 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:04,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:04,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:05,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:05,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:05,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:06,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:07,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:07,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:07,849 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:09,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:09,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:09,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:11,277 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:11,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:12,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:12,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:13,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:14,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:14,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:15,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:17,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:17,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:17,253 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:17,852 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:19,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:19,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:19,853 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:19,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:21,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:22,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:22,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:22,255 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:23,854 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:24,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:24,141 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:25,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:27,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:27,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:27,856 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:27,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:29,856 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:30,852 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:44:30,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:30,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:31,857 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:33,858 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:33,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:33,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:33,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:34,452 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:44:34,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:34,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:34,884 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:35,859 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:35,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:35,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:37,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:38,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:38,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:39,520 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:39,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:40,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:40,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:41,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:44:41,195 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:44:41,195 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:44:41,196 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:44:41,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:41,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:44:43,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:43,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:43,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:45,511 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:45,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:45,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:45,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:47,864 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:48,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:48,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:49,865 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:49,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:49,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:49,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:50,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:50,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:51,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:51,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:53,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:53,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:53,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:55,867 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:55,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:55,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:56,139 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:57,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:44:59,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:59,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:59,869 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:01,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:01,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:01,143 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:01,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:03,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:04,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:04,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:04,455 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:45:04,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:04,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:04,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:05,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:06,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:06,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:07,133 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:07,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:09,603 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:45:09,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:09,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:09,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:11,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:12,133 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:12,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:12,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:13,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:14,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:14,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:15,875 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:17,137 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:17,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:17,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:17,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:19,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:19,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:19,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:19,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:21,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:22,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:22,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:22,780 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:22,794 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:45:22,795 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:45:22,796 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:45:22,796 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:45:22,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:45:23,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:24,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:24,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:25,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:27,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:27,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:27,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:28,780 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:29,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:29,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:29,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:31,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:32,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:32,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:33,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:34,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:34,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:34,423 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:34,456 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:45:34,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:34,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:34,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:35,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:37,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:37,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:37,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:39,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:39,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:40,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:42,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:42,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:44,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:44,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:45,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:47,891 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:45:47,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:47,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:45:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:49,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:49,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:50,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:50,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:50,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:51,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:52,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:52,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:53,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:55,722 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:55,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:55,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:55,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:57,891 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:45:57,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:57,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:59,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:00,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:00,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:01,365 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:01,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:02,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:02,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:03,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:04,458 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:46:04,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:04,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:04,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:05,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:05,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:05,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:07,351 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:07,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:07,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:07,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:09,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:10,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:10,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:11,896 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:12,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:12,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:12,992 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:13,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:15,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:15,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:15,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:16,324 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:46:16,325 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:46:16,326 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:46:16,326 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:46:16,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:46:17,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:17,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:17,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:18,974 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:19,884 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:19,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:19,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:20,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:20,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:21,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:22,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:22,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:23,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:24,626 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:25,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:26,366 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:46:26,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:26,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:27,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:29,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:29,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:29,676 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:29,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:31,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:31,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:31,904 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:33,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:34,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:34,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:34,460 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:46:34,688 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:34,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:34,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:34,885 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:35,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:36,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:36,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:38,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:39,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:39,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:39,706 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:40,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:41,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:41,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:42,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:44,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:44,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:44,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:45,355 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:46,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:46,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:46,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:48,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:49,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:49,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:49,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:50,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:51,269 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:51,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:51,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:54,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:54,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:54,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:56,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:56,529 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:56,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:56,913 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:58,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:46:59,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:59,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:59,838 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:46:59,840 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:46:59,840 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:46:59,841 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:46:59,914 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:47:00,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:01,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:01,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:02,495 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:02,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:02,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:04,462 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:47:04,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:04,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:04,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:04,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:06,472 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:47:06,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:06,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:06,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:08,105 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:08,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:09,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:09,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:10,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:11,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:11,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:12,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:14,063 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:14,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:14,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:15,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:16,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:16,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:17,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:19,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:19,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:19,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:19,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:19,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:20,774 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:21,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:21,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:21,923 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:23,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:24,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:24,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:25,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:25,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:26,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:26,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:27,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:29,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:29,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:29,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:31,204 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:31,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:31,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:31,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:33,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:34,463 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:47:34,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:34,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:34,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:34,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:34,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:35,928 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:36,611 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:36,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:36,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:37,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:39,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:39,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:39,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:41,631 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:41,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:41,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:41,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:43,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:44,565 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:47:44,567 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:47:44,567 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:47:44,568 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:47:44,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:47:45,348 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:47:45,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:45,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:45,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:47,355 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:47,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:48,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:48,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:49,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:49,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:49,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:50,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:50,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:51,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:52,761 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:53,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:53,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:53,935 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:55,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:55,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:55,936 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:57,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:47:58,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:58,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:58,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:59,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:00,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:00,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:01,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:03,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:03,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:03,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:03,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:04,465 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:48:04,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:04,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:04,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:05,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:05,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:05,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:07,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:08,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:08,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:08,718 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:09,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:10,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:10,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:11,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:13,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:13,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:13,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:14,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:15,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:15,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:15,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:17,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:18,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:18,283 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:19,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:19,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:19,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:19,944 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:20,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:20,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:21,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:23,847 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:48:23,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:23,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:23,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:25,175 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:25,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:26,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:26,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:27,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:28,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:28,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:29,201 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:48:29,222 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:48:29,223 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:48:29,223 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:48:29,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:29,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:48:30,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:30,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:30,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:31,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:33,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:33,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:33,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:34,466 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:48:34,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:34,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:34,886 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:35,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:35,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:35,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:35,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:37,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:38,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:38,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:39,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:40,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:40,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:41,794 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:41,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:43,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:43,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:44,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:45,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:46,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:46,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:47,438 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:47,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:48,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:48,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:49,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:49,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:49,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:49,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:50,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:50,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:51,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:52,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:52,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:53,420 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:53,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:55,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:55,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:56,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:57,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:48:58,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:58,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:58,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:59,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:01,680 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:49:01,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:01,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:01,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:03,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:04,468 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:49:04,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:04,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:04,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:04,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:04,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:04,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:05,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:06,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:06,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:07,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:09,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:09,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:09,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:09,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:10,492 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:49:10,494 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:49:10,494 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:49:10,494 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:49:10,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:49:11,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:11,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:11,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:13,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:14,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:14,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:15,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:15,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:16,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:16,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:17,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:19,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:19,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:19,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:19,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:21,663 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:21,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:21,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:23,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:24,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:24,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:25,968 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:26,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:26,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:26,947 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:27,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:28,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:28,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:29,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:31,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:31,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:31,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:32,937 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:33,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:33,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:33,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:34,470 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:49:34,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:34,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:34,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:35,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:36,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:36,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:37,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:38,552 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:39,973 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:39,975 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:49:39,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:40,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:41,973 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:42,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:42,953 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:43,974 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:44,530 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:44,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:44,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:45,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:47,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:48,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:48,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:49,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:49,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:49,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:49,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:50,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:50,048 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:50,101 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:51,841 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:49:51,843 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:49:51,843 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:49:51,844 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:49:51,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:51,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:49:52,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:52,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:52,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:53,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:54,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:54,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:55,129 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:55,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:56,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:56,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:57,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:49:58,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:58,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:59,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:01,118 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:01,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:01,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:01,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:03,981 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:03,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:04,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:04,472 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:50:04,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:04,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:04,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:05,982 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:06,758 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:06,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:06,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:07,982 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:08,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:08,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:09,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:11,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:11,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:11,984 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:12,742 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:13,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:13,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:14,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:15,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:17,438 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:50:17,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:17,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:17,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:18,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:19,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:19,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:19,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:20,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:20,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:21,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:22,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:22,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:23,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:24,342 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:25,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:25,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:25,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:27,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:27,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:27,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:29,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:29,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:30,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:30,316 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:31,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:32,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:32,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:33,281 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:50:33,283 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:50:33,283 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:50:33,283 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:50:33,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:33,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:50:34,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:34,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:34,474 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:50:34,886 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:34,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:34,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:50:34,887 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:34,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:35,057 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:35,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:37,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:37,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:37,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:39,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:39,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:39,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:40,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:41,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:42,325 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:42,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:43,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:44,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:44,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:45,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:46,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:47,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:47,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:47,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:49,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:49,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:50:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:49,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:49,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:51,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:52,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:52,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:52,482 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:53,998 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:55,999 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:56,117 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:50:56,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:56,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:57,999 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:50:58,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:58,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:58,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:00,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:01,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:01,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:02,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:03,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:03,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:04,001 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:04,094 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:04,475 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:51:04,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:04,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:51:04,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:04,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:06,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:06,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:06,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:08,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:08,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:08,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:09,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:10,003 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:11,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:11,120 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:12,003 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:13,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:13,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:14,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:14,717 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:51:14,718 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:51:14,719 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:51:14,719 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:51:15,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:51:15,720 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:16,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:16,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:16,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:17,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:18,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:18,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:18,203 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:19,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:19,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:51:19,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:19,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:20,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:20,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:20,259 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:21,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:22,006 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:23,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:23,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:24,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:25,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:25,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:26,007 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:26,312 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:28,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:28,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:28,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:30,008 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:30,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:30,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:32,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:32,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:33,791 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:51:33,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:34,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:34,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:34,477 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:51:34,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:34,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:34,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:36,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:36,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:36,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:37,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:38,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:38,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:38,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:40,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:41,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:41,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:42,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:43,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:43,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:43,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:44,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:46,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:46,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:46,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:48,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:48,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:49,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:49,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:49,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:50,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:51,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:51,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:52,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:54,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:54,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:54,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:54,590 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:56,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:56,263 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:51:56,265 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:51:56,265 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:51:56,266 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:51:56,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:56,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:57,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:51:58,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:59,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:51:59,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:59,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:59,593 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:00,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:01,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:01,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:02,019 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:03,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:03,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:04,019 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:04,478 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:52:04,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:52:04,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:04,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:04,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:05,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:06,020 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:06,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:06,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:08,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:08,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:10,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:11,764 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:52:11,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:11,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:14,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:14,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:15,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:16,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:16,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:18,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:19,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:19,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:19,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:19,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:19,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:20,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:20,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:21,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:21,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:22,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:24,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:24,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:24,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:26,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:26,360 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:26,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:26,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:28,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:28,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:28,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:30,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:31,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:31,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:32,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:32,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:33,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:33,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:34,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:34,480 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:52:34,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:34,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:36,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:36,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:36,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:37,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:38,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:38,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:38,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:40,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:41,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:41,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:42,032 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:43,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:43,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:43,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:44,032 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:46,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:46,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:46,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:48,034 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:49,545 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:49,559 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:52:49,560 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:52:49,561 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:52:49,561 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:52:49,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:49,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:49,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:49,993 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:52:50,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:50,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:50,034 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:50,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:52:51,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:52,035 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:52,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:52,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:54,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:55,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:55,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:55,531 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:56,037 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:57,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:57,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:58,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:52:59,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:59,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:00,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:01,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:01,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:02,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:02,039 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:04,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:04,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:04,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:04,482 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:53:04,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:04,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:06,041 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:06,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:06,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:06,189 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:08,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:08,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:08,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:10,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:10,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:10,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:12,043 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:12,160 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:12,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:12,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:14,044 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:15,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:15,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:16,044 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:17,791 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:18,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:18,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:18,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:19,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:19,888 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:19,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:20,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:20,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:20,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:22,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:22,827 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:23,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:23,164 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:24,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:26,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:26,770 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:53:26,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:26,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:28,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:28,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:29,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:29,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:30,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:31,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:53:31,157 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:53:31,158 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:53:31,158 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:53:31,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:31,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:32,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:32,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:53:33,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:34,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:34,465 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:34,483 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:53:34,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:34,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:34,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:53:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:34,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:36,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:36,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:36,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:38,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:39,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:39,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:40,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:40,443 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:41,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:41,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:42,054 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:44,055 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:44,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:44,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:46,055 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:46,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:46,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:46,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:48,056 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:49,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:49,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:53:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:49,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:50,056 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:51,178 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:51,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:51,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:52,057 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:54,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:54,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:54,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:56,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:56,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:56,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:57,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:58,059 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:53:59,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:59,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:00,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:01,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:01,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:02,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:02,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:04,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:04,484 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:54:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:54:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:04,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:05,480 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:54:05,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:05,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:06,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:08,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:08,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:08,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:08,628 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:10,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:10,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:10,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:12,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:12,604 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:54:12,605 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:54:12,606 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:54:12,606 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:54:13,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:54:13,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:13,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:14,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:14,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:15,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:15,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:15,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:16,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:18,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:18,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:18,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:19,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:19,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:54:19,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:19,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:20,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:20,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:20,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:20,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:22,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:23,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:23,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:24,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:25,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:25,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:25,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:26,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:28,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:28,338 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:28,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:30,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:30,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:30,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:30,835 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:32,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:33,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:33,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:34,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:34,486 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:54:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:54:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:34,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:35,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:35,426 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:36,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:36,827 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:38,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:38,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:38,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:40,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:40,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:40,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:42,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:42,433 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:44,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:45,010 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:54:45,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:45,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:46,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:47,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:47,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:48,078 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:48,425 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:49,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:50,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:50,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:50,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:52,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:52,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:52,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:54,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:54,054 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:54:54,055 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:54:54,056 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:54:54,056 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:54:54,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:54,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:54:55,076 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:55,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:55,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:56,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:54:57,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:57,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:58,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:00,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:00,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:00,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:00,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:02,083 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:02,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:02,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:04,084 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:04,488 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:55:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:55:04,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:05,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:05,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:05,123 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:06,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:07,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:07,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:08,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:10,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:10,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:10,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:10,599 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:12,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:12,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:12,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:14,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:14,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:14,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:16,088 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:16,561 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:16,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:16,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:18,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:19,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:19,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:19,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:55:19,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:20,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:22,090 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:22,182 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:22,599 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:55:22,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:22,629 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:24,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:25,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:25,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:26,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:27,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:27,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:28,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:28,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:30,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:30,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:30,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:32,094 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:32,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:32,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:33,787 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:34,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:34,490 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:55:34,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:34,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:35,464 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:55:35,466 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:55:35,466 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:55:35,467 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:55:35,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:35,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:36,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:36,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:55:37,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:37,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:37,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:38,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:39,753 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:40,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:40,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:40,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:42,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:42,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:42,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:44,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:45,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:45,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:45,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:46,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:47,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:47,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:48,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:49,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:49,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:49,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:50,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:50,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:50,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:50,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:52,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:52,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:52,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:54,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:55,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:55,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:56,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:55:56,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:57,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:57,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:58,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:00,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:01,176 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:56:01,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:01,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:01,862 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:02,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:03,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:03,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:04,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:04,492 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:56:04,889 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:04,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:06,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:06,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:06,112 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:07,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:08,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:08,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:08,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:10,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:10,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:10,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:12,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:12,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:12,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:13,424 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:14,117 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:15,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:15,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:16,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:16,753 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:56:16,755 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:56:16,755 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:56:16,756 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:56:17,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:56:17,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:18,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:18,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:19,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:19,395 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:19,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:20,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:20,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:20,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:22,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:22,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:22,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:24,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:24,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:24,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:25,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:26,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:27,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:27,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:28,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:29,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:29,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:30,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:30,984 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:32,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:32,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:32,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:34,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:34,493 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:56:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:34,890 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:34,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:34,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:36,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:36,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:38,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:38,529 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:56:38,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:38,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:40,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:41,437 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:41,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:41,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:42,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:43,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:43,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:44,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:46,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:46,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:46,412 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:47,413 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:48,131 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:48,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:48,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:49,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:49,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:49,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:50,132 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:51,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:51,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:52,133 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:53,041 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:53,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:53,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:54,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:56,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:56,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:56,320 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:58,022 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:56:58,023 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:56:58,024 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:56:58,024 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:56:58,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:56:58,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:56:58,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:58,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:59,025 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:59,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:00,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:01,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:01,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:02,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:03,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:03,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:04,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:04,495 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:57:04,497 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:04,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:06,138 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:06,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:06,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:08,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:08,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:08,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:09,601 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:10,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:11,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:11,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:12,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:13,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:13,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:14,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:14,623 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:16,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:17,062 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:57:17,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:17,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:18,143 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:19,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:19,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:19,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:19,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:20,143 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:20,797 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:22,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:22,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:22,144 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:24,145 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:25,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:25,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:26,136 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:26,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:27,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:27,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:28,147 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:30,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:30,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:30,147 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:32,133 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:32,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:32,148 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:32,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:34,149 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:34,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:34,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:34,497 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:57:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:34,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:34,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:36,150 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:36,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:36,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:37,764 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:38,150 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:38,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:38,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:39,449 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:57:39,451 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:57:39,451 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:57:39,452 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:57:40,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:40,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:57:41,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:41,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:41,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:42,152 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:43,758 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:43,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:43,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:44,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:45,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:45,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:46,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:47,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:47,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:48,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:49,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:49,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:49,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:49,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:50,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:50,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:50,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:52,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:54,156 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:54,436 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:57:54,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:54,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:55,354 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:56,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:56,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:56,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:58,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:57:59,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:59,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:00,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:00,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:01,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:01,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:02,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:04,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:04,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:04,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:04,647 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:58:04,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:04,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:04,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:06,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:06,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:06,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:06,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:08,161 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:09,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:09,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:10,162 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:11,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:11,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:12,163 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:12,543 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:14,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:14,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:15,164 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:16,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:16,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:17,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:18,497 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:19,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:19,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:19,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:19,999 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:20,000 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:20,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:20,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:58:21,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:21,345 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:58:21,346 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:58:21,346 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:58:22,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:22,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:22,166 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:58:23,167 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:24,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:24,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:24,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:25,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:26,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:27,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:29,034 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:29,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:29,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:33,532 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:58:33,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:33,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:34,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:34,501 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:58:34,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 16:58:34,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:34,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:34,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:35,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:35,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:38,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:38,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:39,172 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:39,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:40,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:40,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:41,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:43,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:43,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:43,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:45,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:45,537 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:45,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:45,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:47,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:48,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:48,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:49,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:49,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:49,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:49,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:50,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:50,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:51,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:51,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:53,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:53,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:53,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:55,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:55,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:55,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:57,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:57,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:58:58,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:58,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:59,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:00,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:00,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:01,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:02,732 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:03,181 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:03,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:03,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:04,502 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:59:04,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:04,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:04,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:05,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:05,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:05,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:07,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:08,514 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:08,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:08,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:09,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:11,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:12,384 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:59:12,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:12,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:13,185 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:14,139 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:14,153 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:59:14,155 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:59:14,155 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:59:14,156 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:59:14,185 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:59:15,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:15,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:15,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:17,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:17,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:17,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:19,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:19,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:19,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:19,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:20,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:20,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:21,150 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:21,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:22,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:22,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:23,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:25,189 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:25,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:25,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:26,706 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:27,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:27,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:27,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:29,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:29,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:29,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:31,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:32,340 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:32,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:32,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:33,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:34,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:34,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:34,504 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 16:59:34,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:34,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:34,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:35,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:37,194 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:37,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:37,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:38,298 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:39,194 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:39,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:39,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:41,195 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:42,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:42,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:43,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:43,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:44,447 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:44,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:45,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:46,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:46,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:47,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:49,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:49,791 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:59:49,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:49,838 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:49,888 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:49,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:49,891 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:49,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:51,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:52,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:52,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:53,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:54,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:54,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:55,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:55,510 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:55,523 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:59:55,524 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 16:59:55,525 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:59:55,525 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:59:56,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 16:59:57,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:57,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:57,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:59,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 16:59:59,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:59,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:01,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:01,466 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:02,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:02,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:03,204 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:04,506 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:00:04,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:04,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:04,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:04,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:04,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:05,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:07,094 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:07,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:07,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:07,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:09,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:09,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:09,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:11,207 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:12,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:12,840 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:13,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:13,208 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:14,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:14,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:15,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:17,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:17,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:17,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:18,671 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:19,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:19,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:19,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:19,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:19,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:19,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:21,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:22,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:22,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:23,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:24,643 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:24,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:24,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:25,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:27,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:28,435 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:00:28,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:28,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:29,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:30,248 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:30,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:30,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:31,214 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:32,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:32,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:33,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:34,507 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:00:34,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:34,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:34,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:35,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:35,281 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:35,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:35,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:36,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:00:36,956 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:00:36,957 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:00:36,958 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:00:37,216 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:37,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:00:37,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:37,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:39,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:40,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:40,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:41,218 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:41,270 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:42,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:42,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:43,218 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:45,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:45,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:45,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:46,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:47,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:47,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:47,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:49,220 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:49,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:49,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:50,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:50,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:51,220 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:52,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:52,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:52,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:53,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:55,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:55,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:55,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:57,223 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:00:57,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:57,659 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:58,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:59,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:00,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:00,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:01,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:02,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:02,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:03,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:04,466 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:04,509 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:01:04,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:04,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:04,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:05,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:06,132 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:01:06,153 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:06,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:07,227 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:08,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:08,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:09,228 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:10,088 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:10,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:10,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:11,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:12,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:12,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:13,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:15,230 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:15,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:15,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:16,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:17,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:17,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:17,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:18,365 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:01:18,366 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:01:18,367 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:01:18,367 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:01:19,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:19,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:01:19,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:19,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:19,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:20,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:20,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:21,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:21,242 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:22,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:22,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:23,234 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:25,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:25,698 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:25,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:26,633 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:27,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:27,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:27,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:29,236 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:30,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:30,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:31,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:32,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:32,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:32,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:33,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:34,511 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:01:34,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:34,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:34,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:34,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:34,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:35,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:37,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:37,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:37,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:38,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:39,239 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:39,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:39,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:41,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:42,971 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:01:43,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:43,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:43,241 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:44,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:45,241 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:45,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:45,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:47,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:47,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:47,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:49,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:49,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:49,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:49,892 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:49,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:50,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:50,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:51,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:52,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:52,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:53,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:55,245 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:55,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:55,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:55,781 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:57,246 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:57,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:57,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:59,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:01:59,782 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:01:59,782 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:01:59,783 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:01:59,783 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:02:00,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:02:00,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:00,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:00,784 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:01,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:02,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:02,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:03,248 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:04,513 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:02:04,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:04,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:04,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:05,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:05,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:05,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:06,066 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:07,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:07,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:07,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:09,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:10,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:10,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:11,251 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:11,417 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:12,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:12,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:13,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:15,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:15,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:15,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:17,254 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:17,415 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:17,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:18,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:19,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:19,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:19,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:19,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:21,256 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:21,576 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:02:21,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:21,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:23,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:23,256 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:24,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:24,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:25,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:26,547 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:27,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:27,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:28,990 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:29,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:29,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:29,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:31,259 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:32,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:32,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:33,260 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:34,514 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:02:34,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:34,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:34,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:34,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:34,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:34,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:35,260 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:37,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:37,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:37,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:39,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:39,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:39,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:40,328 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:41,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:41,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:02:41,339 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:02:41,339 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:02:41,340 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:02:42,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:02:42,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:42,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:43,263 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:44,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:44,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:45,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:45,832 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:47,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:47,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:47,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:49,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:49,892 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:49,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:49,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:51,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:51,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:52,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:52,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:53,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:54,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:54,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:55,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:56,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:57,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:02:57,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:57,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:59,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:01,243 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:03:01,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:01,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:01,984 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:04,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:04,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:04,686 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:03:04,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:04,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:04,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:05,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:06,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:06,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:07,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:07,639 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:09,272 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:09,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:09,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:11,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:11,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:12,273 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:13,278 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:14,274 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:14,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:14,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:16,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:16,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:16,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:18,276 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:18,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:18,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:19,274 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:19,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:19,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:19,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:20,276 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:21,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:21,412 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:22,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:23,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:23,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:24,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:24,287 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:24,964 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:03:24,965 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:03:24,966 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:03:24,966 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:03:25,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:03:26,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:26,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:26,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:27,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:28,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:28,436 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:28,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:29,301 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:30,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:31,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:31,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:32,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:33,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:33,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:34,281 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:34,517 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:03:34,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:34,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:34,893 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:34,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:36,281 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:36,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:36,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:38,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:39,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:40,136 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:03:40,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:40,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:40,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:42,283 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:43,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:43,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:44,283 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:45,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:45,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:46,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:46,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:48,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:48,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:48,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:49,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:49,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:50,285 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:50,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:50,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:51,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:52,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:52,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:52,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:54,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:55,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:55,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:56,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:56,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:03:57,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:57,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:58,288 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:00,289 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:00,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:00,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:01,273 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:02,290 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:02,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:02,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:04,290 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:04,519 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:04:04,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:04,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:04,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:05,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:05,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:06,291 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:06,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:07,202 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:04:07,204 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:04:07,204 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:04:07,205 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:04:07,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:04:07,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:07,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:08,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:09,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:10,293 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:10,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:10,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:11,576 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:12,293 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:12,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:12,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:14,294 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:15,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:15,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:16,295 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:17,258 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:18,296 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:19,693 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:04:19,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:19,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:19,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:19,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:19,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:20,296 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:22,297 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:22,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:22,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:23,242 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:24,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:24,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:24,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:26,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:27,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:27,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:28,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:28,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:29,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:29,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:30,300 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:32,300 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:32,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:32,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:34,301 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:34,521 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:04:34,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:34,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:34,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:34,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:34,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:34,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:36,302 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:37,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:37,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:38,303 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:39,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:39,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:39,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:40,304 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:42,306 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:42,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:42,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:44,306 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:44,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:44,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:45,459 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:46,307 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:47,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:47,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:48,308 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:04:49,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:49,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:04:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:49,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:49,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:50,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:52,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:52,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:54,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:54,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:55,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:58,238 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:04:58,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:58,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:00,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:05:00,198 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:05:00,198 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:05:00,199 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:05:00,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:05:01,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:01,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:01,200 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:01,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:02,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:03,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:03,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:04,313 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:04,522 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:05:04,893 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:04,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:04,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:06,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:06,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:06,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:08,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:08,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:08,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:10,315 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:11,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:11,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:12,244 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:12,315 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:13,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:13,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:14,316 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:16,317 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:16,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:16,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:18,241 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:18,318 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:18,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:18,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:19,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:19,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:19,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:20,319 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:21,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:21,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:22,319 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:23,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:23,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:23,860 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:24,320 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:26,320 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:26,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:26,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:28,321 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:28,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:28,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:29,837 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:30,322 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:31,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:31,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:32,322 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:33,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:33,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:34,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:34,523 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:05:34,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:34,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:34,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:35,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:36,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:36,799 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:05:37,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:37,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:38,324 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:40,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:40,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:40,325 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:40,453 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:42,106 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:05:42,107 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:05:42,107 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:05:42,108 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:05:42,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:42,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:05:42,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:42,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:43,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:44,327 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:44,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:44,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:46,327 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:46,432 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:47,953 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:47,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:48,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:49,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:49,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:49,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:50,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:50,329 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:51,707 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:52,330 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:52,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:53,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:54,330 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:55,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:55,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:56,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:05:57,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:58,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:58,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:58,332 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:00,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:00,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:00,332 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:02,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:02,333 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:03,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:03,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:04,334 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:04,525 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:06:04,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:04,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:04,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:05,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:05,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:06,335 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:08,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:08,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:08,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:08,336 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:10,336 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:10,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:10,395 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:12,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:13,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:13,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:13,699 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:14,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:16,339 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:16,909 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:06:16,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:16,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:18,340 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:19,657 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:19,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:19,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:20,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:20,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:20,340 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:22,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:22,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:22,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:23,645 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:06:23,647 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:06:23,648 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:06:23,648 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:06:24,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:24,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:06:24,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:24,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:25,287 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:25,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:26,343 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:26,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:26,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:28,344 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:29,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:29,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:30,344 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:31,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:31,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:31,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:32,345 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:34,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:34,526 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:06:34,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:34,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:34,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:34,894 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:34,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:36,347 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:36,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:36,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:36,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:38,347 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:39,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:39,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:40,348 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:42,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:42,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:42,349 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:42,868 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:44,350 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:44,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:44,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:46,350 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:47,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:47,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:48,351 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:48,486 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:49,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:49,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:49,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:50,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:50,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:50,352 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:52,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:52,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:52,353 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:54,353 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:54,461 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:55,600 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:06:55,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:55,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:56,354 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:58,355 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:06:58,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:58,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:00,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:00,356 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:00,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:00,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:02,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:03,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:03,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:04,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:04,528 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:07:04,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:04,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:04,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:05,084 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:07:05,085 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:07:05,085 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:07:05,085 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:07:05,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:05,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:05,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:05,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:07:06,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:06,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:08,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:08,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:08,359 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:10,360 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:10,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:10,397 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:11,708 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:12,361 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:13,366 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:13,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:14,362 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:15,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:15,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:16,362 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:17,681 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:18,363 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:18,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:18,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:19,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:19,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:20,364 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:20,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:20,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:22,365 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:23,311 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:23,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:23,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:24,365 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:25,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:25,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:26,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:28,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:28,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:28,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:29,293 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:30,367 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:30,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:30,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:32,368 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:34,274 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:07:34,368 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:34,530 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:07:34,531 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:34,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:34,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:34,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:35,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:35,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:36,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:37,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:37,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:38,370 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:39,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:39,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:39,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:40,370 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:41,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:41,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:42,371 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:43,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:44,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:44,372 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:45,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:46,372 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:46,573 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:07:46,575 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:07:46,575 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:07:46,576 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:07:47,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:47,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:47,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:47,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:07:48,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:49,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:49,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:49,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:49,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:49,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:50,374 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:51,079 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:51,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:51,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:52,374 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:54,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:54,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:54,375 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:56,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:56,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:56,376 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:07:56,534 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:58,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:58,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:58,377 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:00,377 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:00,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:00,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:02,378 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:02,499 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:02,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:02,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:04,379 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:04,531 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:08:04,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:04,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:04,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:05,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:05,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:06,380 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:07,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:07,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:08,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:08,380 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:10,381 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:11,499 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:08:11,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:11,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:12,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:14,115 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:14,383 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:14,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:14,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:16,384 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:16,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:16,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:18,384 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:19,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:19,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:19,737 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:19,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:19,895 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:20,385 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:21,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:21,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:22,386 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:24,387 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:24,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:24,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:24,790 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:26,388 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:26,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:26,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:28,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:08:28,127 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:08:28,127 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:08:28,128 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:08:28,388 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:28,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:08:29,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:29,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:29,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:30,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:30,776 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:31,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:31,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:32,390 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:33,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:33,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:34,391 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:34,532 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:08:34,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:34,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:34,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:36,115 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:36,392 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:36,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:36,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:38,393 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:38,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:38,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:40,393 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:41,400 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:41,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:41,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:42,394 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:43,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:43,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:44,395 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:46,396 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:46,421 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:46,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:46,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:48,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:49,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:49,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:49,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:50,344 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:08:50,384 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:50,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:50,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:52,079 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:52,398 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:53,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:53,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:54,399 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:55,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:55,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:56,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:57,631 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:58,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:08:58,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:58,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:00,401 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:00,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:00,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:02,402 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:03,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:03,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:03,602 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:04,402 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:04,535 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:09:04,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:04,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:04,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:05,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:05,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:06,403 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:07,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:07,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:08,404 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:09,245 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:09,259 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:09:09,260 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:09:09,260 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:09:09,261 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:09:09,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:09,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:09:10,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:10,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:10,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:12,406 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:12,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:12,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:14,406 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:14,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:14,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:15,212 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:16,407 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:17,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:17,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:18,408 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:19,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:19,336 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:19,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:19,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:19,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:20,409 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:20,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:21,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:21,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:22,409 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:24,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:24,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:24,410 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:26,411 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:26,831 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:27,862 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:09:27,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:27,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:28,411 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:30,412 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:30,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:30,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:32,413 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:32,461 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:32,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:32,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:34,414 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:34,536 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:09:34,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:34,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:35,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:35,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:36,414 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:37,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:37,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:38,415 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:38,453 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:39,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:39,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:40,416 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:42,417 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:42,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:42,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:44,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:44,418 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:44,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:44,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:46,418 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:47,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:47,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:48,419 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:49,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:49,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:49,881 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:49,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:49,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:49,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:50,420 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:50,554 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:09:50,556 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:09:50,556 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:09:50,557 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:09:51,420 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:51,420 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:09:52,421 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:52,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:52,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:54,422 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:54,882 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:54,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:54,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:56,422 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:57,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:57,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:58,423 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:09:59,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:59,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:00,424 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:00,554 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:02,425 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:02,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:02,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:04,425 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:04,538 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:10:04,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:04,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:04,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:06,426 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:06,531 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:06,702 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:10:06,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:06,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:08,427 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:09,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:09,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:10,427 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:11,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:11,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:12,157 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:12,428 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:14,429 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:14,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:14,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:16,430 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:17,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:17,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:18,117 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:18,430 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:19,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:19,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:19,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:20,074 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:20,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:20,431 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:22,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:22,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:22,432 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:23,747 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:24,433 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:25,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:25,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:26,433 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:27,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:27,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:28,434 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:29,716 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:30,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:30,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:30,435 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:32,045 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:10:32,047 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:10:32,047 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:10:32,048 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:10:32,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:32,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:32,435 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:32,435 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:10:34,436 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:34,539 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:10:34,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:34,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:34,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:35,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:35,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:35,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:36,437 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:37,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:37,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:38,437 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:39,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:39,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:40,315 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:40,438 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:42,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:42,425 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:42,438 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:44,439 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:46,138 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:10:46,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:46,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:46,275 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:46,440 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:48,440 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:49,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:49,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:49,896 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:49,896 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:49,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:50,441 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:51,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:51,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:51,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:52,442 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:53,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:53,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:54,444 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:56,444 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:56,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:56,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:57,878 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:58,445 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:10:58,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:58,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:00,446 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:00,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:00,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:02,446 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:03,518 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:03,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:03,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:04,447 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:04,541 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:11:04,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:04,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:04,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:05,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:05,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:06,448 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:07,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:07,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:08,449 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:08,771 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:10,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:10,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:12,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:13,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:13,771 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:15,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:15,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:18,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:18,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:18,772 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:19,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:20,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:20,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:20,453 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:22,454 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:23,918 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:11:23,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:23,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:23,983 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:24,454 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:26,656 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:11:26,658 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:11:26,658 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:11:26,659 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:11:26,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:26,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:27,455 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:11:28,456 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:28,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:28,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:29,661 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:31,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:31,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:34,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:34,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:34,542 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:11:34,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:34,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:34,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:35,090 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:36,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:36,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:38,305 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:38,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:38,459 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:40,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:41,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:41,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:43,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:43,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:45,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:45,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:45,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:48,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:48,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:49,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:49,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:49,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:50,140 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:50,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:50,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:52,463 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:11:52,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:52,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:55,294 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:55,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:55,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:57,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:57,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:00,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:01,199 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:12:01,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:01,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:04,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:04,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:04,544 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:12:04,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:04,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:04,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:06,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:06,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:06,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:06,469 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:12:08,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:08,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:11,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:11,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:11,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:13,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:13,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:15,769 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:15,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:16,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:18,472 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:12:18,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:18,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:19,897 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:19,897 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:19,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:20,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:20,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:21,222 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:24,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:24,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:26,223 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:26,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:26,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:28,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:28,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:30,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:30,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:31,224 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:32,477 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:12:33,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:33,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:34,545 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:12:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:12:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:34,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:35,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:35,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:37,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:39,113 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:12:39,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:39,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:42,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:42,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:42,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:44,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:44,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:46,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:46,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:46,481 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:12:47,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:49,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:49,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:49,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:49,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:12:49,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:49,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:51,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:51,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:53,117 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:54,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:54,698 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:56,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:56,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:58,486 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:12:58,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:58,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:59,005 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:01,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:01,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:03,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:04,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:04,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:04,547 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:13:04,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:04,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:13:04,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:04,898 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:06,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:06,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:09,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:09,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:09,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:11,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:11,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:13,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:13,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:13,491 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:13:14,265 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:16,744 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:13:16,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:16,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:19,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:19,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:19,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:19,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:19,898 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:19,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:21,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:21,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:23,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:23,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:24,413 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:25,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:25,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:27,495 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:13:28,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:28,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:29,420 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:30,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:30,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:32,888 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:32,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:34,422 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:34,550 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:13:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:34,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:34,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:13:34,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:35,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:35,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:37,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:37,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:39,501 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:13:39,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:39,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:39,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:43,045 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:43,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:44,824 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:45,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:45,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:47,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:47,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:49,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:49,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:49,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:49,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:50,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:50,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:53,505 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:13:53,722 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:13:53,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:53,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:55,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:55,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:55,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:58,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:58,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:00,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:00,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:00,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:02,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:02,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:04,552 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:14:04,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:04,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:04,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:05,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:05,508 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:14:05,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:05,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:08,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:08,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:10,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:10,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:10,105 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:13,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:13,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:15,106 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:15,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:15,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:17,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:17,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:19,513 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:14:19,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:19,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:19,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:20,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:20,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:21,187 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:22,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:22,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:25,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:25,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:26,187 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:28,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:28,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:31,455 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:32,035 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:14:32,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:32,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:33,517 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:14:34,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:34,120 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:34,553 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:14:34,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:34,899 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:34,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:37,040 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:37,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:37,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:39,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:39,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:41,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:41,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:42,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:44,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:44,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:45,521 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:14:46,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:46,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:47,782 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:48,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:48,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:49,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:49,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:14:49,900 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:50,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:51,461 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:53,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:53,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:53,617 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:55,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:55,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:57,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:57,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:59,479 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:59,526 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:15:00,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:00,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:02,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:02,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:04,480 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:04,554 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:15:04,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:04,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:04,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:04,900 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:07,983 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:15:08,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:08,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:10,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:11,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:11,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:13,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:13,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:13,530 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:15:15,817 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:16,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:16,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:18,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:19,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:19,899 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:20,234 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:20,235 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:21,418 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:22,594 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:22,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:24,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:25,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:26,752 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:27,534 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:15:29,245 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:29,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:31,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:31,753 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:32,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:34,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:34,814 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:15:34,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:34,900 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:35,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:37,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:38,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:39,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:41,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:41,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:41,539 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:15:42,970 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:44,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:44,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:46,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:46,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:47,970 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:49,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:49,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:49,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:49,900 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:49,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:53,026 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:15:53,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:53,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:53,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:55,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:55,118 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:55,544 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:15:58,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:58,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:58,945 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:00,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:00,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:02,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:02,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:03,946 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:04,558 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:16:04,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:04,900 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:05,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:05,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:07,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:07,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:09,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:09,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:09,219 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:11,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:11,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:11,549 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:16:14,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:14,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:14,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:16,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:16,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:18,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:18,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:19,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:19,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:19,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:19,901 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:21,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:21,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:23,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:23,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:24,291 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:25,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:25,554 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:16:25,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:29,334 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:29,369 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:16:29,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:29,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:32,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:32,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:34,334 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:34,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:34,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:34,559 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:16:34,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:34,901 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:34,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:36,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:36,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:39,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:39,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:39,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:41,559 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:16:41,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:41,595 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:43,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:43,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:44,594 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:46,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:46,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:48,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:48,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:49,595 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:49,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:49,901 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:50,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:50,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:53,563 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:16:53,934 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:53,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:54,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:56,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:56,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:58,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:58,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:59,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:01,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:01,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:03,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:03,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:04,561 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:17:04,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:04,901 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:05,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:06,897 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:17:06,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:06,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:07,567 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:17:09,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:09,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:10,370 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:11,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:11,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:13,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:13,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:15,370 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:16,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:16,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:19,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:19,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:19,900 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:19,901 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:19,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:20,532 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:21,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:21,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:21,571 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:17:23,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:23,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:25,533 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:26,178 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:26,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:28,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:28,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:30,533 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:31,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:31,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:33,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:33,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:33,575 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:17:34,564 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:17:34,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:34,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:35,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:35,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:36,129 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:38,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:38,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:40,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:40,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:41,129 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:44,412 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:17:44,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:44,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:46,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:47,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:47,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:47,578 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:17:49,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:49,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:49,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:49,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:51,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:51,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:52,080 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:54,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:54,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:56,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:56,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:57,081 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:58,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:58,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:01,584 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:18:01,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:01,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:02,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:04,021 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:04,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:04,565 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:18:04,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:04,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:06,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:06,089 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:08,090 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:09,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:09,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:11,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:11,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:13,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:13,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:13,368 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:13,587 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:18:16,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:16,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:18,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:18,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:18,370 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:19,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:19,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:19,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:21,807 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:18:21,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:21,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:23,713 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:24,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:25,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,593 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:18:29,685 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:29,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:30,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:32,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:32,075 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:34,566 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:18:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:34,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:34,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:35,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:35,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:37,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:37,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:39,981 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:40,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:40,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:41,597 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:18:42,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:42,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:44,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:44,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:45,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:46,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:46,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:49,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:49,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:49,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:18:49,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:51,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:51,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:51,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:53,602 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:18:54,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:54,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:56,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:56,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:56,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:00,637 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:19:00,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:00,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:01,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:03,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:03,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:04,568 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:19:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:04,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:04,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:05,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:05,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:06,227 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:07,606 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:19:07,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:07,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:10,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:10,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:11,228 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:13,017 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:13,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:15,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:15,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:16,229 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:18,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:18,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:19,611 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:19:19,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:19,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:19,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:20,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:20,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:22,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:23,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:23,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:25,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:25,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:27,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:27,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:27,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:30,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:30,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,726 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:33,616 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:19:34,569 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:19:34,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:34,648 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:34,902 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:34,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:38,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:38,110 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:19:38,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:38,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:41,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:41,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:43,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:43,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:43,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:45,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:45,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:45,620 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:19:48,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:48,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:48,786 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:49,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:49,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:49,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:50,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:50,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:52,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:52,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:54,739 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:55,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:55,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:57,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:57,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:59,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:59,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:59,624 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:20:00,138 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:02,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:02,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:04,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:04,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:04,571 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:20:04,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:04,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:04,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:06,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:07,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:07,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:09,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:09,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:11,029 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:11,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:11,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:13,628 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:20:15,418 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:20:15,464 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:15,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:16,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:18,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:18,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:19,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:19,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:19,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:20,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:20,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:22,031 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:22,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:22,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:25,632 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:20:25,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:25,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:27,638 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:27,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:27,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:29,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:29,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:32,640 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:32,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:32,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:34,573 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:20:34,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:34,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:34,902 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:34,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:34,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:36,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:36,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:38,544 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:39,637 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:20:39,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:39,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:41,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:41,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:43,545 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:43,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:43,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:46,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:46,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:48,546 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:48,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:48,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:49,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:49,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:49,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:51,641 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:20:52,631 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:20:52,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:52,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:53,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:55,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:55,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:57,684 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:57,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:58,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:00,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:00,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:02,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:03,001 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:03,713 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:04,575 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:21:04,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:04,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:04,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:05,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:05,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:05,645 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:21:08,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:08,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:09,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:10,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:10,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:12,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:12,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:14,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:15,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:15,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:17,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:17,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:19,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:19,650 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:21:19,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:19,903 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:19,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:20,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:20,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:22,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:22,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:25,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:25,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:25,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:27,799 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:27,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:30,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:31,546 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:21:31,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:31,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:31,655 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:21:34,576 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:21:34,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:34,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:34,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:34,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:34,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:35,129 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:36,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:36,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:38,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:38,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:40,129 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:41,736 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:41,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:43,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:43,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:45,659 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:21:45,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:45,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:45,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:48,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:48,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:49,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:49,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:49,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:50,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:50,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:51,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:53,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:53,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:55,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:55,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:56,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:58,018 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:58,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:59,663 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:22:00,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:00,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:01,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:03,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:03,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:04,578 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:22:04,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:04,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:05,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:05,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:07,098 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:08,780 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:22:08,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:08,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:11,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:11,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:13,012 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:13,668 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:22:14,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:14,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:16,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:16,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:18,013 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:19,105 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:19,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:19,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:19,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:21,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:21,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:23,398 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:24,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:24,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:25,671 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:22:26,410 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:26,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:28,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:28,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:29,274 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:31,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:31,539 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:33,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:33,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:34,275 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:34,580 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:22:34,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:34,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:34,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:35,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:35,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:38,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:38,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:39,676 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:22:39,745 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:40,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:40,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:43,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:43,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:44,745 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:47,300 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:22:47,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:47,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:49,746 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:49,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:49,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:49,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:50,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:50,350 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:52,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:52,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:53,681 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:22:55,111 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:55,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:55,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:57,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:57,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:00,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:00,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:00,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:02,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:02,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:04,581 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:23:04,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:04,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:05,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:05,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:05,684 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:23:06,042 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:07,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:07,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:10,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:10,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:11,043 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:12,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:12,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:14,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:14,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:16,044 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:16,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:16,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:19,687 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:23:19,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:19,904 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:19,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:19,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:21,119 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:21,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:21,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:25,469 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:23:25,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:25,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:26,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:28,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:28,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:30,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:30,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:31,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:33,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:33,273 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:33,691 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:23:34,583 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:23:34,904 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:34,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:35,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:35,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:37,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:38,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:38,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:40,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:40,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:42,050 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:43,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:43,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:45,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:45,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:45,694 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:23:47,678 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:48,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:48,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:49,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:49,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:49,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:50,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:50,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:53,082 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:53,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:53,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:55,330 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:55,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:58,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:58,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:58,637 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:59,699 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:00,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:00,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:04,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:04,585 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:24:04,850 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:24:04,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:04,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:04,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:04,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:04,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:05,701 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:06,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:06,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:09,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:09,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:10,341 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:11,703 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:11,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:11,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:14,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:14,656 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:15,342 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:16,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:16,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:19,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:19,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:19,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:19,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:21,124 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:21,533 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:21,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:23,707 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:24,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:24,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:26,306 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:26,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:26,412 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:29,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:29,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:31,307 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:31,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:31,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:34,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:34,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:34,586 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:24:34,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:34,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:34,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:36,456 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:36,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:36,597 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:37,712 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:39,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:39,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:41,598 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:42,949 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:24:42,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:42,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:46,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:46,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:46,600 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:48,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:48,056 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:49,716 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:24:49,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:49,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:49,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:51,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:51,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:52,117 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:53,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:53,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:55,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:55,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:57,118 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:57,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:57,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:00,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:00,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:02,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:02,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:02,992 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:03,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:25:04,588 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:25:04,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:04,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:05,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:05,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:07,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:07,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:08,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:10,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:10,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:12,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:12,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:13,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:15,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:15,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:17,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:17,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:17,724 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:25:18,429 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:19,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:19,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:21,190 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:25:21,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:21,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:24,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:24,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:24,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:26,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:26,089 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:28,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:28,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:29,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:29,728 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:25:30,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:30,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:33,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:33,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:34,590 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:25:34,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:34,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:34,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:35,087 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:35,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:35,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:38,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:38,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:40,088 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:40,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:41,006 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:43,732 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:25:43,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:43,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:45,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:46,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:46,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:48,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:48,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:49,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:49,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:49,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:50,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:50,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:51,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:53,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:53,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:55,735 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:25:55,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:55,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:57,054 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:59,229 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:25:59,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:59,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:02,055 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:02,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:02,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:04,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:04,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:04,591 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:26:04,905 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:04,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:06,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:06,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:07,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:08,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:09,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:09,739 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:26:12,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:12,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:12,153 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:14,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:14,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:17,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:17,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:17,154 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:19,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:19,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:19,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:22,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:22,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:22,424 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:23,744 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:26:24,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:24,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:27,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:27,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:27,426 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:29,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:29,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:31,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:31,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:32,426 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:33,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:33,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:34,593 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:26:34,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:34,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:34,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:35,746 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:26:37,276 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:26:37,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:37,316 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:38,046 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:40,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:40,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:42,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:42,263 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:43,047 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:45,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:45,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:47,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:47,391 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:48,733 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:49,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:26:49,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:49,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:49,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:50,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:50,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:52,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:52,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:54,178 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:55,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:55,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:57,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:57,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:59,180 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:59,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:59,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:01,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:27:01,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:01,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:04,594 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:27:04,595 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:04,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:04,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:04,906 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:04,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:06,862 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:06,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:09,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:09,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:10,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:11,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:11,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:15,079 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:27:15,096 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:15,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:15,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:15,759 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:27:17,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:17,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:19,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:19,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:19,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:20,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:22,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:22,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:24,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:24,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:25,102 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:27,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:27,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:29,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:29,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:29,763 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:27:30,422 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:32,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:32,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:34,596 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:27:34,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:34,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:34,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:34,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:36,048 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:37,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:37,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:39,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:39,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:41,429 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:41,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:41,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:41,767 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:27:44,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:44,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:46,430 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:46,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:46,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:49,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:49,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:49,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:49,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:49,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:52,076 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:52,887 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:27:52,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:52,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:55,772 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:27:55,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:55,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:57,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:57,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:57,964 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:00,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:00,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,710 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:04,598 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:28:04,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:04,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:04,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:05,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:05,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:07,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:07,581 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:07,776 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:28:08,069 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:10,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:10,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:12,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:12,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:13,070 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:15,503 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:15,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:17,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:17,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:18,071 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:19,906 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:19,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:20,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:20,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:21,780 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:28:22,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:22,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:23,084 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:25,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:25,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:27,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:27,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:28,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:30,691 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:28:30,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:30,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:33,089 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:33,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:33,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:33,783 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:28:34,599 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:28:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:34,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:35,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:35,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:38,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:38,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:39,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:40,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:40,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:43,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:43,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:44,087 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:45,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:45,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:48,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:48,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:48,788 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:28:49,101 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:49,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:49,907 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:49,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:50,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:50,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:53,148 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:53,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:54,127 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:55,194 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:55,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:57,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:58,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:59,128 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:00,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:00,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:00,792 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:29:02,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:03,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:04,210 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:04,601 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:29:04,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:04,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:04,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:05,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:05,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:08,489 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:29:08,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:08,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:10,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:11,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:11,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:13,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:13,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:14,797 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:29:15,417 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:16,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:16,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:18,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:18,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:19,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:20,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:20,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:21,159 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:22,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:22,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:25,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:25,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:26,569 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:27,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:27,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:28,802 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:29:30,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:30,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:31,570 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:32,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:32,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:34,603 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:29:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:34,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:35,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:35,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:37,082 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:37,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:37,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:40,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:40,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:40,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:29:42,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:42,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:42,842 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:46,013 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:29:46,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:46,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:47,842 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:48,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:48,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:49,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:49,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:49,909 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:50,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:50,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:53,004 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:53,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:53,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:54,809 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:29:55,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:55,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:58,005 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:58,515 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:58,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:00,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:00,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:03,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:03,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:03,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:04,605 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:30:04,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:04,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:04,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:05,459 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:05,474 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:06,812 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:30:08,074 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:08,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:08,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:10,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:10,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:13,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:13,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:13,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:15,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:15,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:18,075 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:18,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:18,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:19,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:19,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:19,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:20,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:20,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:20,816 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:30:23,099 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:23,779 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:30:23,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:23,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:26,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:26,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:28,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:28,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:28,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:31,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:31,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:32,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:30:33,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:33,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:33,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:34,606 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:30:34,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:34,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:34,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:36,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:36,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:38,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:38,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:39,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:41,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:41,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:43,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:43,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:44,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:46,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:46,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:46,822 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:30:48,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:48,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:49,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:49,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:49,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:49,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:51,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:51,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:53,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:53,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:55,118 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:56,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:56,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:58,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:58,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:58,826 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:31:00,415 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:01,640 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:31:01,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:01,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:04,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:04,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:04,607 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:31:04,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:04,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:04,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:06,130 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:06,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:06,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:09,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:09,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,582 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,721 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:12,829 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:31:14,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:14,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:16,722 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:17,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:17,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:19,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:19,909 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:19,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:20,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:20,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:22,131 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:22,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:22,244 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:25,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:25,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:26,834 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:31:27,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:27,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:27,758 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:29,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:29,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:31,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:31,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:32,758 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:34,608 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:31:34,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:34,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:34,908 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:34,908 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:34,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:36,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:36,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:38,542 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:38,838 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:31:40,063 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:31:40,085 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:40,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:42,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:42,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:43,542 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:44,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:44,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:47,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:47,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:48,543 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:49,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:49,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:49,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:49,909 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:49,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:50,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:31:52,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:52,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:53,749 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:54,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:54,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:57,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:57,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:58,750 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:59,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:59,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:02,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:02,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:04,517 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:04,610 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:32:04,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:04,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:04,845 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:32:04,909 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:04,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:04,910 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:07,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:07,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:09,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:09,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:10,099 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:12,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:12,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:14,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:14,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:15,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:16,848 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:32:18,078 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:32:18,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:18,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:19,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:32:19,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:19,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:19,911 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:20,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:20,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:20,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:22,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:22,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,822 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:27,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:27,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:30,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:30,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:30,851 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:32:31,592 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:32,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:32,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:34,612 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:32:34,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:34,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:34,910 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:35,528 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:35,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:37,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:37,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:37,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:40,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:40,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:42,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:32:45,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:45,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,381 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,611 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:49,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:49,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:32:49,910 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:49,911 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:50,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:50,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:52,194 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:52,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:52,620 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:55,646 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:32:55,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:55,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:56,860 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:32:57,845 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:58,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:58,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:00,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:00,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:02,845 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:03,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:03,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:03,601 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:33:03,603 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:33:03,603 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:33:03,604 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:33:03,862 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:33:04,614 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:33:04,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:04,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:33:04,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:04,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:04,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:05,408 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:05,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:05,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:08,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:08,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:08,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:10,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:10,364 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:12,866 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:13,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:13,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:13,806 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:15,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:15,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:18,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:18,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:18,807 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:19,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:20,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:20,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:33:20,182 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:20,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:20,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:22,869 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:23,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:23,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:23,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:24,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:25,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:25,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:26,870 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:28,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:28,718 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:28,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:29,772 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:30,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:30,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:30,872 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:32,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:34,358 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:33:34,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:34,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:34,616 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:33:34,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:34,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:34,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:34,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:35,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:36,874 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:37,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:37,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:38,875 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:39,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:39,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:40,357 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:40,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:42,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:42,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:42,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:44,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:44,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:44,877 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:46,322 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:46,878 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:47,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:47,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:48,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:49,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:49,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:49,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:49,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:49,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:50,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:51,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:51,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:51,953 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:52,880 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:53,996 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:54,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:54,881 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:56,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:56,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:56,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:56,955 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:58,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:33:59,024 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:59,037 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:00,883 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:01,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:01,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:02,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:02,938 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:03,617 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:34:03,618 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:34:03,619 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:34:03,619 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:34:03,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:34:04,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:04,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:04,618 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:34:04,884 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:04,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:04,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:04,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:05,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:06,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:06,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:06,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:08,585 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:08,886 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:08,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:08,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:10,887 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:12,589 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:34:12,609 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:12,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:12,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:14,575 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:14,643 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:14,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:14,888 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:16,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:17,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:17,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:18,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:19,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:19,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:19,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:19,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:19,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:20,091 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:20,890 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:21,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:21,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:22,891 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:24,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:24,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:24,892 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:25,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:26,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:26,282 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:26,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:28,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:29,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:29,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:30,894 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:31,139 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:31,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:31,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:32,895 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:34,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:34,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:34,619 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:34:34,896 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:34,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:34,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:34,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:36,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:36,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:36,768 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:36,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:38,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:39,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:39,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:40,898 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:41,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:41,294 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:42,745 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:42,899 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:43,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:43,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:44,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:45,083 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:34:45,084 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:34:45,084 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:34:45,085 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:34:45,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:34:46,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:46,438 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:46,900 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:47,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:48,420 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:48,901 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:49,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:49,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:49,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:50,062 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:34:50,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:50,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:50,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:52,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:52,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:52,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:54,420 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:54,903 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:55,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:55,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:56,904 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:57,180 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:57,190 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:58,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:34:59,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:59,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:00,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:00,905 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:02,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:02,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:02,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:04,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:04,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:04,621 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:35:04,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:04,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:04,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:04,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:05,118 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:06,907 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:07,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:07,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:08,908 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:09,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:09,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:10,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:10,991 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:12,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:12,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:12,910 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:14,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:14,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:14,911 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:16,625 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:16,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:17,111 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:17,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:18,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:19,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:19,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:19,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:19,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:19,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:22,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:22,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:22,404 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:22,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:24,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:24,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:24,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:26,368 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:35:26,370 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:35:26,370 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:35:26,370 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:35:26,917 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:26,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:35:27,751 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:35:27,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:27,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:27,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:28,032 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:28,918 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:30,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:30,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:30,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:32,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:32,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:32,919 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:33,994 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:34,623 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:35:34,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:34,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:34,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:34,920 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:35,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:35,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:36,921 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:37,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:37,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:38,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:39,654 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:40,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:40,757 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:40,922 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:42,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:42,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:42,923 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:44,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:44,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:44,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:45,646 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:46,924 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:47,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:47,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:48,925 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:49,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:49,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:49,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:49,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:50,926 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:51,137 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:51,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:51,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:52,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:54,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:54,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:54,927 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:56,305 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:56,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:56,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:56,928 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:58,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:35:59,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:59,669 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:00,929 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:01,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:01,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:02,277 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:02,930 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:04,624 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:36:04,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:04,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:04,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:04,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:05,494 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:36:06,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:06,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:06,931 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:07,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:07,907 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:36:07,909 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:36:07,909 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:36:07,910 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:36:07,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:36:08,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:08,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:08,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:09,932 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:10,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:11,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:11,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:12,933 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:13,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:13,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:13,866 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:14,934 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:16,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:16,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:16,935 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:18,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:18,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:18,935 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:19,494 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:19,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:19,912 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:19,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:20,936 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:21,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:21,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:22,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:23,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:23,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:24,937 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:25,468 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:26,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:26,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:26,938 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:28,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:28,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:28,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:30,939 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:31,096 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:31,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:31,774 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:32,940 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:33,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:33,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:34,626 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:36:34,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:34,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:34,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:34,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:36,691 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:36,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:36,941 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:37,061 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:38,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:38,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:38,942 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:40,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:40,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:40,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:42,687 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:42,943 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:44,103 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:36:44,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:44,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:44,944 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:46,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:46,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:46,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:48,659 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:48,945 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:48,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:49,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:49,330 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:36:49,331 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:36:49,331 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:36:49,332 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:36:49,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:49,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:49,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:36:50,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:51,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:51,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:51,946 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:52,947 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:53,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:53,940 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:54,302 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:54,947 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:55,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:55,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:56,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:36:58,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:58,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:58,949 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:00,254 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:00,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:01,238 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:01,249 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:02,950 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:03,269 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:03,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:04,628 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:37:04,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:04,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:04,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:04,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:05,301 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:05,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:05,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:06,952 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:08,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:08,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:08,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:10,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:10,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:10,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:11,891 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:12,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:13,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:13,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:14,955 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:15,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:15,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:16,956 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:17,495 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:18,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:18,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:18,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:19,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:19,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:19,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:20,957 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:21,786 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:37:21,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:21,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:22,958 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:23,246 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:24,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:24,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:24,959 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:26,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:26,793 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:26,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:28,875 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:28,960 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:29,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:29,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:30,546 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:37:30,547 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:37:30,548 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:37:30,548 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:37:30,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:30,961 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:37:31,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:31,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:31,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:32,962 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:34,630 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:37:34,631 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:34,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:34,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:34,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:34,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:34,963 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:34,968 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:36,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:37,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:37,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:38,964 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:39,823 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:39,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:39,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:40,965 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:41,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:41,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:42,966 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:44,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:44,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:44,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:45,457 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:46,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:46,967 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:46,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:48,968 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:49,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:49,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:50,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:50,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:50,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:50,969 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:52,097 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:52,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:52,970 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:54,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:55,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:55,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:56,391 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:56,971 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:37:57,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:57,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:58,972 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:00,674 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:38:00,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:00,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:00,973 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:02,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:02,974 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:03,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:03,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:04,631 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:38:04,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:04,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:04,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:04,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:05,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:05,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:06,975 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:07,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:07,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:07,991 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:08,976 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:10,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:10,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:10,977 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:12,000 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:38:12,001 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:38:12,002 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:38:12,002 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:38:12,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:12,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:12,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:12,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:38:13,003 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:13,978 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:14,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:15,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:15,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:16,979 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:17,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:17,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:18,621 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:18,980 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:19,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:19,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:19,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:20,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:20,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:20,981 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:22,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:22,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:22,982 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:24,598 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:24,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:25,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:25,805 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:26,983 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:27,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:27,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:28,984 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:29,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:29,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:30,238 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:30,985 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:32,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:32,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:32,986 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:34,633 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:38:34,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:34,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:34,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:34,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:34,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:34,987 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:36,211 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:36,988 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:38,241 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:38:38,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:38,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:38,989 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:40,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:41,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:41,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:41,871 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:42,990 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:43,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:43,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:44,991 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:46,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:46,206 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:46,992 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:47,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:48,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:48,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:48,993 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:49,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:49,913 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:50,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:51,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:51,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:52,994 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:53,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:53,221 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:53,495 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:53,508 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:38:53,509 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:38:53,510 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:38:53,510 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:38:53,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:38:54,995 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:55,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:56,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:56,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:56,996 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:58,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:58,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:58,997 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:38:59,485 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:00,998 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:01,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:01,107 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:02,998 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:03,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:03,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:04,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:04,635 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:39:04,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:04,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:04,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:04,999 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:06,161 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:06,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:07,000 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:08,204 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:08,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:09,001 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:10,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:11,001 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:11,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:11,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:13,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:13,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:13,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:15,002 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:16,100 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:17,003 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:17,032 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:39:17,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:17,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:19,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:19,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:19,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:19,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:19,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:19,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:21,004 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:21,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:21,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:21,933 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:23,005 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:24,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:24,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:25,006 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:26,321 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:27,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:27,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:29,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:29,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:31,322 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:32,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:32,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:34,101 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:34,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:34,637 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:39:34,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:39:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:34,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:37,009 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:37,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:37,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:37,202 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:39,010 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:39,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:39,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:41,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:42,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:42,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:42,568 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:43,011 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:44,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:44,261 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:45,012 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:46,554 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:39:46,555 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:39:46,556 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:39:46,556 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:39:47,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:47,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:39:47,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:47,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:48,013 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:48,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:49,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:49,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:49,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:49,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:51,014 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:51,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:51,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:53,015 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:54,149 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:55,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:55,892 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:39:55,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:55,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:57,016 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:58,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:58,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:59,017 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:39:59,766 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:01,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:01,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:01,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:03,018 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:03,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:03,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:04,639 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:40:04,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:04,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:04,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:05,020 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:05,119 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:05,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:05,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:08,021 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:08,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:08,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:10,022 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:10,699 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:10,978 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:10,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:12,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:14,023 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:14,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:14,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,024 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:16,179 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,318 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:18,025 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:19,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:19,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:19,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:19,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:19,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:20,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:21,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:21,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:22,026 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:22,282 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:24,027 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:24,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:24,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:26,028 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:26,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:26,191 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:27,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:27,941 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:40:27,942 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:40:27,943 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:40:27,943 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:40:28,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:28,029 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:40:29,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:29,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:30,030 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:31,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:31,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:32,031 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:33,898 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:34,033 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:34,640 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:40:34,877 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:40:34,897 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:34,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:34,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:34,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:36,036 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:37,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:37,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:38,038 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:39,532 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:39,712 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:39,733 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:40,040 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:42,042 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:42,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:42,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:44,045 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:44,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:44,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:45,520 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:46,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:47,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:47,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:48,046 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:49,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:49,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:49,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:49,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:50,047 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:50,808 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:52,048 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:52,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:52,892 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:54,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:54,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:54,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:56,049 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:56,155 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:57,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:57,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:58,050 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:40:59,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:00,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:00,051 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:02,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:02,135 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:02,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:03,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:04,052 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:04,642 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:41:04,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:04,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:04,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:05,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:05,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:06,053 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:07,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:08,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:08,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:08,054 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:09,426 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:41:09,427 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:41:09,427 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:41:09,428 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:41:10,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:10,055 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:10,055 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:41:10,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:12,056 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:13,619 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:41:13,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:13,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:13,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:14,057 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:16,057 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:16,402 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:16,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:18,058 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:18,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:18,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:19,344 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:19,914 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:19,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:19,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:20,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:21,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:21,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:22,060 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:23,392 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:23,403 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:24,061 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:25,320 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:25,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:25,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:26,062 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:28,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:28,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:28,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:30,063 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:30,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:30,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:30,948 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:32,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:33,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:33,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:34,064 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:34,644 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:41:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:34,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:35,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:35,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:36,065 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:36,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:38,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:38,181 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:38,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:40,066 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:40,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:40,248 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:42,067 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:42,573 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:43,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:43,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:44,068 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:45,302 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:45,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:46,069 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:48,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:48,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:48,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:48,531 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:49,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:50,070 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:50,885 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:41:51,418 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:41:51,418 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:41:51,418 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:41:51,928 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:41:51,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:51,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:52,071 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:52,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:41:54,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:54,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:55,050 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:55,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:56,072 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:41:57,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:57,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:58,073 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:00,074 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:00,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:00,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:00,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:02,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:02,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:02,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:04,075 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:04,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:04,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:04,645 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:42:04,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:04,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:04,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:05,768 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:06,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:06,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:06,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:08,076 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:08,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:08,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:10,077 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:11,342 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:11,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:11,742 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:12,078 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:13,388 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:13,405 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:14,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:16,079 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:16,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:16,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:17,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:18,080 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:18,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:18,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:19,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:19,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:19,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:20,081 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:20,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:20,361 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:22,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:22,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:22,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:23,347 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:24,082 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:25,229 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:25,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:26,083 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:28,084 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:28,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:29,058 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:42:29,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:29,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:30,085 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:31,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:31,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:32,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:32,309 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:42:32,310 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:42:32,311 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:42:32,311 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:42:33,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:42:34,086 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:34,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:34,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:34,648 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:42:34,651 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:34,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:34,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:36,087 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:36,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:36,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:38,088 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:38,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:38,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:39,943 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:40,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:41,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:41,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:42,089 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:43,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:43,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:44,090 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:45,337 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:46,091 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:46,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:46,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:48,092 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:48,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:48,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:49,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:49,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:49,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:50,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:51,325 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:51,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:51,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:52,093 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:53,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:53,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:54,094 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:55,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:55,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:56,095 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:56,959 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:58,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:42:58,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:59,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:00,096 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:01,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:01,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:02,097 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:02,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:04,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:04,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:04,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:04,648 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:43:04,915 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:04,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:06,098 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:07,788 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:43:07,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:07,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:08,099 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:08,569 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:10,100 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:10,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:10,705 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:12,101 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:12,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:12,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:13,573 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:43:13,575 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:43:13,576 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:43:13,576 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:13,576 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:43:14,101 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:14,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:43:14,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:14,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:16,102 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:17,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:17,891 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:18,103 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:19,537 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:19,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:19,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:19,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:19,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:19,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:20,104 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:21,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:21,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:22,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:24,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:24,016 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:24,105 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:25,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:26,106 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:26,816 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:26,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:28,107 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:28,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:28,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:30,108 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:31,156 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:31,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:31,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:32,109 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:33,988 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:34,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:34,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:34,650 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:43:34,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:34,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:34,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:36,110 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:36,795 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:37,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:37,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:38,111 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:39,236 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:39,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:40,112 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:42,113 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:42,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:42,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:42,788 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:44,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:46,114 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:46,314 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:43:46,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:46,352 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:47,793 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:48,115 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:48,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:48,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:49,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:49,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:49,917 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:50,116 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:50,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:51,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:52,890 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:53,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:53,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:54,117 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:55,144 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:43:55,145 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:43:55,145 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:43:55,146 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:43:55,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:55,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:56,118 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:56,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:43:58,119 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:43:58,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:58,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:58,442 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:00,120 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:00,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:00,285 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:02,121 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:02,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:02,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:03,452 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:04,121 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:04,652 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:44:04,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:04,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:04,916 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:05,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:05,460 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:06,122 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:07,485 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:07,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:08,123 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:08,454 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:09,521 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:09,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:10,124 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:11,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:11,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:12,125 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:14,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:14,440 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:14,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:14,641 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:16,126 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:16,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:16,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:18,127 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:19,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:19,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:19,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:19,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:19,917 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:20,128 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:20,722 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:22,129 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:23,386 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:44:23,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:23,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:24,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:26,116 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:26,130 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:26,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:26,501 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:28,131 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:28,524 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:28,535 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:30,132 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:31,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:31,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:31,792 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:32,133 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:33,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:33,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:34,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:34,654 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:44:34,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:44:34,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:34,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:34,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:35,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:35,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:36,134 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:36,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:36,812 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:44:36,814 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:44:36,814 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:44:36,814 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:44:37,135 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:44:38,136 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:38,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:38,822 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:40,137 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:40,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:40,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:42,138 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:42,794 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:43,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:43,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:44,139 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:45,845 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:45,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:46,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:47,883 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:47,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:48,140 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:48,461 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:49,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:44:49,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:49,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:49,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:50,141 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:50,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:50,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:52,142 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:52,803 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:52,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:54,143 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:54,447 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:54,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:54,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:56,144 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:56,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:56,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:58,144 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:44:59,459 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:00,145 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:00,655 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:45:00,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:00,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:02,146 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:03,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:03,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:04,147 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:04,655 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:45:04,656 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:04,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:45:04,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:04,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:04,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:05,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:05,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:06,148 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:07,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:08,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:08,148 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:10,115 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:10,149 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:10,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:10,784 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:12,150 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:12,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:12,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:14,151 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:15,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:15,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:15,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:16,152 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:17,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:17,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:18,152 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:18,269 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:45:18,271 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:45:18,271 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:45:18,271 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:45:19,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:45:19,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:19,782 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:19,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:45:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:19,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:20,153 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:21,400 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:22,154 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:22,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:22,550 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:24,155 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:24,570 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:24,588 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:26,156 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:26,446 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:26,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:26,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:28,157 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:28,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:28,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:31,512 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:31,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:31,741 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:32,158 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:33,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:33,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:34,159 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:34,657 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:45:34,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:45:34,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:34,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:34,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:36,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:37,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:37,173 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:45:37,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:37,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:38,160 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:40,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:40,121 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:42,063 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:42,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:42,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:44,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:45,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:47,029 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:47,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:47,064 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:49,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:45:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:49,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:49,993 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:50,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:50,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:52,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:52,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:52,165 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:52,178 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:54,070 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:54,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:56,108 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:56,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:56,167 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:57,332 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:58,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:45:59,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:59,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:00,168 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:01,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:01,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:02,169 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:02,405 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:03,977 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:46:03,978 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:46:03,979 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:46:03,980 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:46:04,129 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:04,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:04,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:04,170 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:46:04,658 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:46:04,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:04,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:06,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:06,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:06,201 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:08,171 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:08,234 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:09,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:09,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:10,172 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:11,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:11,232 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:12,173 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:13,743 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:14,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:14,936 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:46:14,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:14,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:16,174 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:18,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:18,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:18,175 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:19,243 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:19,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:46:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:19,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:20,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:20,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:20,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:22,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:22,104 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:22,176 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:24,177 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:24,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:24,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:24,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:26,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:26,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:26,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:28,178 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:29,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:29,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:30,179 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:30,207 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:31,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:31,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:32,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:34,180 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:34,660 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:46:34,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:34,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:34,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:34,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:46:34,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:34,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:35,725 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:36,181 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:36,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:36,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:38,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:39,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:40,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:40,182 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:41,504 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:42,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:42,086 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:42,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:43,158 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:46:43,160 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:46:43,160 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:46:43,160 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:46:43,183 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:46:44,149 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:44,168 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:44,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:46,184 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:46,225 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:46,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:47,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:48,185 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:49,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:49,193 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:46:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:49,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:50,186 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:52,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:52,678 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:52,741 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:46:52,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:52,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:54,187 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:55,670 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:55,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:56,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:57,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:57,716 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:57,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:58,188 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:46:59,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:59,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:00,189 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:02,190 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:02,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:02,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:03,417 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:04,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:04,661 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:47:04,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:04,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:47:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:04,918 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:06,191 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:07,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:07,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:08,192 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:09,015 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:09,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:10,004 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:10,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:12,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:12,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:12,193 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:14,194 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:14,459 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:15,058 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:15,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:16,195 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:17,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:17,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:18,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:19,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:19,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:19,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:20,196 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:20,564 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:21,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:22,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:22,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:22,198 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:47:22,199 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:47:22,199 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:47:22,200 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:47:23,197 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:47:24,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:24,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:24,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:26,198 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:26,412 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:26,984 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:26,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:28,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:30,199 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:30,603 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:47:30,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:30,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:31,972 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:32,200 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:33,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:33,577 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:34,201 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:34,663 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:47:34,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:34,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:34,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:35,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:35,614 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:36,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:37,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:38,202 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:38,612 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:38,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:40,203 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:40,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:40,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:42,204 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:42,578 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:43,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:43,574 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:44,204 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:45,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:45,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:46,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:48,205 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:48,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:48,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:48,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:49,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:49,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:50,206 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:50,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:50,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:52,207 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:53,564 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:53,565 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:53,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:54,207 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:55,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:55,617 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:56,208 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:58,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:47:58,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:58,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:59,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:00,209 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:00,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:00,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:01,286 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:48:01,287 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:48:01,288 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:48:01,288 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:48:02,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:02,210 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:48:03,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:03,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:04,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:04,665 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:48:04,666 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:04,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:04,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:05,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:05,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:06,211 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:08,212 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:08,972 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:48:09,997 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:10,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:10,195 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:10,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:12,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:12,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:12,213 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:14,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:14,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:14,214 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:15,382 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:16,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:16,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:16,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:18,215 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:18,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:18,810 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:19,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:19,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:19,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:20,216 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:20,901 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:21,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:21,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:22,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:23,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:23,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:24,217 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:26,218 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:26,360 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:26,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:26,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:28,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:28,992 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:29,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:30,219 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:31,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:31,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:31,980 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:32,220 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:33,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:34,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:34,220 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:34,666 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:48:34,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:34,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:34,919 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:36,221 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:36,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:36,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:37,391 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:38,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:38,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:38,850 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:40,222 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:40,456 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:48:40,457 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:48:40,458 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:48:40,458 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:48:41,223 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:48:41,658 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:41,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:42,223 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:42,911 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:43,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:43,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:44,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:46,224 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:47,186 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:48:47,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:47,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:48,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:48,448 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:49,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:49,906 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:49,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:49,920 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:50,225 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:51,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:51,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:52,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:54,209 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:54,226 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:54,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:54,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:56,227 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:56,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:56,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:58,228 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:48:59,468 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:59,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:59,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:00,228 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:01,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:01,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:02,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:04,229 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:04,667 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:49:04,669 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:04,918 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:04,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:04,920 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:04,954 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:04,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:06,230 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:06,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:07,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:08,231 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:09,816 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:09,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:09,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:10,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:12,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:12,014 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:12,232 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:14,233 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:14,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:14,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:15,327 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:16,234 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:16,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:16,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:18,235 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:19,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:19,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:19,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:19,920 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:19,968 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:49:20,144 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:49:20,144 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:49:20,145 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:49:20,236 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:20,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:49:21,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:21,920 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:21,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:22,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:23,237 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:24,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:25,413 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:49:25,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:25,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:26,238 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:26,378 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:28,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:28,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:28,239 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:30,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:30,175 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:30,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:32,240 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:32,266 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:32,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:33,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:34,241 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:34,669 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:49:34,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:34,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:34,920 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:35,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:35,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:36,242 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:37,363 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:37,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:37,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:38,243 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:40,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:40,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:40,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:42,244 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:42,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:42,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:43,255 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:44,245 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:44,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:44,847 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:46,246 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:47,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:47,682 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:48,247 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:48,399 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:49,700 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:49,714 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:49,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:49,920 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:50,248 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:52,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:52,626 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:52,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:54,249 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:54,286 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:54,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:54,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:56,250 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:57,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:57,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:58,251 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:49:59,413 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:59,420 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:49:59,422 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:49:59,422 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:49:59,422 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:49:59,776 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:59,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:00,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:00,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:50:01,252 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:02,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:03,250 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:50:03,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:03,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:04,253 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:04,671 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:50:04,672 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:04,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:04,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:04,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:06,254 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:06,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:06,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:08,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:08,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:08,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:10,176 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:10,255 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:10,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:10,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:12,256 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:13,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:13,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:14,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:15,293 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:15,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:15,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:16,257 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:18,258 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:18,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:18,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:19,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:19,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:20,259 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:20,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:20,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:21,181 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:22,259 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:23,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:23,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:24,260 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:25,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:25,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:26,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:26,295 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:28,261 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:28,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:28,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:30,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:30,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:30,861 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:32,208 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:32,262 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:32,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:32,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:34,263 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:34,672 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:50:34,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:34,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:34,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:36,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:36,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:36,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:37,704 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:38,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:38,089 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:38,264 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:38,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:50:38,913 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:50:38,914 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:50:38,914 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:50:39,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:50:40,265 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:41,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:41,611 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:50:41,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:41,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:42,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:42,967 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:44,266 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:44,423 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:44,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:46,267 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:46,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:46,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:48,268 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:48,499 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:49,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:49,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:49,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:49,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:49,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:50,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:51,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:51,525 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:52,269 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:53,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:53,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:53,969 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:54,270 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:55,580 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:55,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:56,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:58,271 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:50:58,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:58,771 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:59,499 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:00,272 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:00,792 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:00,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:02,273 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:02,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:02,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:04,274 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:04,674 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:51:04,676 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:04,919 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:04,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:05,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:05,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:06,274 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:07,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:07,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:08,275 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:09,854 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:10,276 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:10,487 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:10,499 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:12,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:12,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:12,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:14,277 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:14,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:14,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:15,366 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:16,278 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:18,117 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:51:18,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:18,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:18,260 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:51:18,261 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:51:18,261 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:51:18,262 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:51:18,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:18,279 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:51:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:51:19,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:19,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:20,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:20,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:20,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:21,052 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:21,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:22,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:22,280 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:22,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:24,281 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:24,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:24,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:26,282 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:26,426 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:27,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:27,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:28,283 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:29,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:29,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:30,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:32,097 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:32,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:32,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:32,284 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:34,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:34,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:34,285 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:34,676 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:51:34,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:34,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:34,921 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:36,286 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:36,299 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:36,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:37,763 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:38,287 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:39,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:39,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:41,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:41,537 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:42,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:43,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:43,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:46,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:46,615 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:47,765 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:48,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:48,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:51:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:49,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:49,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:50,290 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:50,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:50,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:52,291 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:52,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:54,292 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:55,054 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:51:55,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:55,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:56,293 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:57,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:57,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:58,294 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:51:58,863 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:59,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:59,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:00,294 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:02,295 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:02,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:02,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:04,296 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:04,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:04,372 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:04,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:04,677 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:52:04,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:04,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:52:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:04,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:06,297 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:06,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:06,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:08,477 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:08,494 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:09,670 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:10,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:11,320 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:52:11,322 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:52:11,322 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:52:11,322 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:52:11,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:11,556 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:12,298 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:12,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:52:13,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:13,605 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:13,618 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:14,299 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:15,565 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:16,300 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:16,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:16,572 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:18,301 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:18,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:18,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:52:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:19,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:20,301 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:20,965 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:21,506 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:21,527 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:22,302 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:23,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:23,603 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:24,303 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:26,304 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:26,654 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:26,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:26,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:28,304 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:28,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:28,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:30,305 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:31,869 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:32,306 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:33,418 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:52:33,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:33,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:34,307 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:34,679 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:52:34,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:34,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:52:34,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:34,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:35,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:35,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:36,307 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:37,718 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:38,308 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:38,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:38,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:40,309 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:40,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:40,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:42,310 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:43,230 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:43,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:43,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:44,311 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:45,297 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:45,311 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:46,311 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:47,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:47,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:48,312 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:48,748 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:49,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:52:49,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:49,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:50,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:50,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:50,313 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:51,024 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:52:51,026 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:52:51,027 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:52:51,027 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:52:51,313 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:52:52,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:52,090 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:52,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:53,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:54,265 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:54,314 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:54,872 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:54,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:56,315 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:56,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:56,917 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:58,316 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:52:59,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:59,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:59,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:00,317 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:01,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:01,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:02,318 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:04,319 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:04,681 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:53:04,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:04,715 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:04,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:04,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:04,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:05,098 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:06,319 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:06,735 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:06,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:08,320 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:10,132 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:53:10,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:10,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:10,231 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:10,321 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:12,321 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:12,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:12,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:14,322 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:14,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:14,909 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:15,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:16,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:17,841 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:17,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:18,323 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:19,873 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:19,882 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:19,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:19,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:53:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:19,922 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:20,324 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:21,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:22,325 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:22,689 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:22,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:24,325 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:24,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:24,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:26,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:26,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:27,554 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:27,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:28,326 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:29,599 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:29,616 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:30,327 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:30,757 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:53:30,759 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:53:30,759 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:53:30,760 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:53:31,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:53:32,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:32,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:32,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:32,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:33,328 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:34,329 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:34,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:34,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:34,683 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:53:34,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:34,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:34,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:36,329 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:37,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:37,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:37,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:38,330 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:39,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:39,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:40,331 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:42,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:42,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:42,332 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:43,379 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:44,284 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:44,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:44,332 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:46,333 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:48,054 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:53:48,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:48,092 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:48,334 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:49,257 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:49,920 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:49,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:49,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:50,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:50,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:50,334 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:52,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:52,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:52,335 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:54,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:54,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:54,336 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:55,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:56,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:56,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:56,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:58,337 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:53:58,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:58,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:00,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:00,566 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:01,704 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:01,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:02,338 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:03,753 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:03,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:04,339 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:04,684 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:54:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:54:04,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:04,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:06,119 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:06,340 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:06,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:06,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:08,340 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:08,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:08,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:10,001 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:54:10,002 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:54:10,003 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:54:10,003 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:54:10,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:10,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:54:10,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:10,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:11,341 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:11,624 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:12,342 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:12,966 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:12,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:14,343 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:16,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:16,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:16,343 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:17,255 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:18,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:18,198 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:18,344 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:19,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:19,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:54:19,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:19,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:20,345 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:21,122 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:21,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:22,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:22,701 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:24,346 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:24,738 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:54:24,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:24,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:26,348 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:27,564 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:27,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:28,596 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:29,350 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:29,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:29,611 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:31,351 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:31,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:31,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:33,352 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:33,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:33,665 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:33,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:34,686 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:54:34,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:34,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:34,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:35,353 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:36,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:36,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:37,354 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:38,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:38,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:39,355 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:39,543 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:40,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:40,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:41,356 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:43,356 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:43,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:43,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:44,707 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:45,357 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:45,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:45,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:47,358 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:48,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:48,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:49,359 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:49,603 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:54:49,604 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:54:49,605 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:54:49,605 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:54:49,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:49,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:49,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:50,056 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:50,359 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:54:50,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:50,830 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:51,360 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:53,360 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:53,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:53,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:55,361 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:55,478 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:55,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:55,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:57,362 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:54:58,000 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:58,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:59,362 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:00,601 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:01,363 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:01,669 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:55:01,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:01,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:03,364 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:04,637 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:04,674 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:04,688 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:55:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:04,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:04,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:05,365 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:06,479 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:06,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:06,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:07,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:09,366 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:09,490 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:09,502 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:11,367 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:11,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:11,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:12,292 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:13,368 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:14,557 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:14,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:15,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:16,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:16,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:17,369 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:17,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:18,677 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:18,695 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:19,370 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:19,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:55:19,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:19,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:21,371 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:21,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:21,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:23,227 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:23,371 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:23,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:23,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:25,372 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:25,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:25,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:27,373 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:28,665 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:28,678 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:55:28,679 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:55:28,680 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:55:28,680 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:55:28,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:28,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:29,374 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:29,374 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:55:30,904 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:30,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:31,374 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:33,375 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:33,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:33,852 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:34,554 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:34,689 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:55:34,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:34,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:55:34,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:34,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:35,376 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:35,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:35,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:37,377 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:39,302 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:55:39,340 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:39,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:39,377 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:39,657 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:41,378 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:42,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:42,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:43,379 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:44,573 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:44,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:45,380 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:45,552 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:46,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:46,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:47,381 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:49,382 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:49,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:49,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:49,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:49,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:49,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:50,634 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:51,383 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:51,857 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:51,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:53,383 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:54,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:54,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:55,384 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:56,522 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:56,886 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:56,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:57,385 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:59,385 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:55:59,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:59,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:01,386 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:01,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:01,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:01,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:03,387 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:03,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:03,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:04,691 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:56:04,921 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:04,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:04,923 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:05,388 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:07,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:07,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:07,388 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:07,566 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:08,205 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:56:08,206 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:56:08,206 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:56:08,207 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:56:08,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:56:09,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:09,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:09,389 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:11,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:11,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:11,390 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:13,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:13,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:13,390 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:14,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:15,391 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:17,392 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:17,853 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:56:17,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:17,880 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:18,200 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:19,393 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:19,899 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:19,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:56:19,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:19,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:21,394 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:22,717 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:22,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:23,394 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:23,434 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:24,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:24,764 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:25,395 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:27,396 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:27,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:27,955 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:29,187 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:29,396 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:29,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:29,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:31,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:32,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:32,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:33,397 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:34,464 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:34,692 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:56:34,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:34,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:34,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:35,182 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:35,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:35,398 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:37,214 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:37,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:37,399 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:39,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:40,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:40,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:40,352 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:41,400 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:42,246 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:42,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:43,401 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:45,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:45,094 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:45,402 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:45,533 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:47,116 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:47,126 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:47,171 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:56:47,173 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:56:47,173 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:56:47,173 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:56:47,403 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:47,403 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:56:49,404 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:49,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:49,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:50,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:50,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:50,826 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:51,404 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:52,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:52,162 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:53,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:55,405 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:55,930 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:56:55,945 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:55,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:55,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:57,406 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:56:57,994 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:58,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:59,407 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:00,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:00,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:01,265 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:01,407 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:02,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:02,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:03,408 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:04,694 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:57:04,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:04,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:04,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:04,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:04,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:05,409 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:07,149 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:07,409 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:07,814 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:07,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:09,410 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:09,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:09,856 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:11,411 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:12,233 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:12,683 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:12,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:13,411 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:14,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:14,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:15,412 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:16,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:16,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:17,413 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:18,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:18,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:18,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:19,413 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:19,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:19,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:21,414 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:22,144 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:22,169 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:23,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:23,415 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:24,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:24,406 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:25,415 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:26,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:57:26,387 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:57:26,388 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:57:26,388 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:57:26,416 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:57:26,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:26,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:27,416 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:29,011 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:29,417 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:29,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:29,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:31,418 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:33,091 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:57:33,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:33,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:33,418 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:34,355 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:34,696 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:57:34,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:34,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:34,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:35,419 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:35,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:35,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:37,420 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:37,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:37,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:39,421 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:40,036 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:41,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:41,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:41,421 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:43,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:43,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:43,422 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:45,613 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:45,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:45,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:47,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:47,946 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:49,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:49,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:57:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:49,924 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:49,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:49,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:51,428 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:52,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:52,846 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:54,871 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:54,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:55,426 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:56,966 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:57,427 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:57,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:57,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:59,428 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:57:59,979 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:59,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:01,967 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:02,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:02,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:03,429 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:04,697 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:58:04,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:04,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:04,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:04,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:04,949 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:05,430 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:06,986 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:07,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:07,430 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:07,874 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:09,431 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:10,713 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:58:10,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:10,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:11,432 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:13,384 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:13,432 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:13,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:13,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:15,433 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:15,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:15,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:17,434 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:17,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:17,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:19,267 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:19,435 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:19,922 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:19,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:19,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:21,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:21,036 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:21,165 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:58:21,179 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:58:21,180 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:58:21,180 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:58:21,435 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:21,436 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:58:23,063 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:23,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:23,436 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:24,409 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:25,437 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:26,023 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:26,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:27,438 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:28,077 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:28,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:29,438 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:30,301 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:31,266 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:31,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:31,439 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:33,365 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:33,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:33,440 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:34,699 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:58:34,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:34,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:34,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:35,438 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:35,441 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:36,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:36,661 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:37,442 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:38,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:38,727 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:39,442 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:40,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:40,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:41,339 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:41,443 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:42,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:42,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:43,445 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:45,445 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:45,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:46,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:46,509 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:47,446 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:49,447 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:49,711 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:58:49,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:49,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:49,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:49,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:51,448 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:51,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:51,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:52,442 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:53,449 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:54,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:54,900 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:55,449 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:56,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:56,931 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:57,450 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:58:57,982 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:58,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:58,963 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:59,451 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:00,903 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:59:00,905 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:59:00,905 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:59:00,906 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:59:01,452 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:01,452 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:59:02,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:02,087 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:03,452 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:03,528 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:04,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:04,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:04,701 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:59:04,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:04,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:05,453 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:07,233 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:07,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:07,454 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:09,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:09,314 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:09,335 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:09,455 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:11,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:11,404 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:11,456 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:13,456 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:14,566 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:14,567 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:14,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:15,457 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:16,650 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:16,664 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:17,458 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:18,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:18,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:19,459 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:19,852 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:19,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:19,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:21,459 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:21,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:21,749 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:23,460 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:23,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:23,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:25,461 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:25,757 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:27,462 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:27,652 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:59:27,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:27,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:29,463 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:30,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:30,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:30,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:31,464 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:32,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:32,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:33,464 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:34,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:34,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:34,703 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 17:59:34,923 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:34,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:35,465 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:36,778 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:37,466 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:37,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:37,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:39,467 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:39,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:39,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:40,372 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:59:40,374 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 17:59:40,374 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:59:40,375 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:59:40,467 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 17:59:41,468 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:41,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:41,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:41,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:43,469 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:44,866 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:44,895 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:45,469 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:46,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:47,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:47,470 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:47,743 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:49,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:49,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:49,471 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 17:59:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:49,925 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:51,472 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:52,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:52,170 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:52,887 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:53,473 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:54,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:54,215 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:55,473 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:56,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:56,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:57,474 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 17:59:58,825 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:59,275 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:59,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:59,475 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:01,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:01,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:01,476 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:03,477 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:04,323 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:04,704 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:00:04,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:00:04,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:04,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:04,955 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:00:04,987 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:04,999 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:05,478 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:07,479 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:07,851 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:07,869 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:09,479 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:09,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:09,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:09,913 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:11,480 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:13,019 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:13,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:13,481 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:15,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:15,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:15,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:15,482 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:17,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:17,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:17,483 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:19,483 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:19,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:00:19,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:19,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:20,005 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 18:00:20,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:20,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:21,179 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 18:00:21,179 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:21,180 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:00:21,180 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:00:21,484 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 18:00:22,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:22,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:23,485 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:24,328 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:24,339 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:25,486 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:26,536 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:27,486 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:27,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:27,598 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:29,487 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:29,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:29,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:31,488 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:31,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:31,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:32,204 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:33,488 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:34,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:34,671 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:34,706 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:00:34,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:00:34,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:34,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:35,489 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:36,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:36,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:37,490 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:37,602 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:38,723 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:38,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:39,491 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:41,491 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:42,362 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:00:42,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:42,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:43,492 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:43,513 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:45,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:45,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:45,493 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:47,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:47,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:47,493 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:48,558 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:49,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:49,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:49,494 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:00:49,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:49,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:52,478 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:52,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:53,495 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:54,458 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:54,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:54,523 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:55,496 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:56,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:56,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:57,497 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:59,498 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:00:59,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:59,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:59,673 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:59,679 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 18:00:59,680 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 18:00:59,680 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:00:59,681 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:01:00,498 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 18:01:01,498 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:01:01,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:01,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:03,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:03,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:04,683 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:04,708 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:01:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:01:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:04,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:06,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:06,832 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:08,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:08,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:10,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:11,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:11,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:13,502 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:01:13,982 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:13,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:16,007 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:17,034 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:17,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:19,924 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:01:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:19,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:20,735 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:01:20,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:20,766 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:21,117 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:22,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:22,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:25,506 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:01:25,823 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:25,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:26,498 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:27,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:27,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:29,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:29,910 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:31,499 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:32,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:32,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:34,709 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:01:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:01:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:34,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:34,983 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:35,007 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:36,969 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:37,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:37,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:37,509 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:01:40,028 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:40,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:41,970 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:42,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:42,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:44,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:44,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:46,970 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:47,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:47,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:49,195 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:49,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:49,513 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:01:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:01:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:49,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:52,083 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:52,331 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:52,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:54,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:54,440 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:57,084 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:58,108 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:01:58,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:58,136 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:01,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:01,224 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:02,954 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:03,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:03,253 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:03,517 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:02:04,711 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:02:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:02:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:04,926 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:05,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:05,288 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:08,119 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:08,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:08,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:10,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:10,327 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:12,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:12,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:13,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:15,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:15,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:15,520 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:02:17,457 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:17,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:18,792 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:19,491 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:19,508 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:02:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:19,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:22,481 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:22,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:23,962 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:24,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:24,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:27,524 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:02:27,652 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:27,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:29,245 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:29,732 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:29,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:31,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:31,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:34,246 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:34,712 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:02:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:02:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:34,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:35,337 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:02:35,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:35,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:38,520 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:38,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:39,527 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:02:39,818 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:40,551 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:40,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:42,591 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:42,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:44,819 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:45,651 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:45,666 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:47,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:47,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:49,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:49,747 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:49,820 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:02:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:49,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:51,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:51,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:53,531 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:02:54,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:54,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:55,388 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:56,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:56,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:59,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:59,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:00,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:01,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:01,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:03,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:03,890 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:04,714 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:03:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:03:04,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:04,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:05,539 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:03:06,104 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:06,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:06,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:08,944 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:08,956 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:11,105 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:12,661 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:03:12,680 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:12,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:15,767 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:15,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:16,106 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:17,544 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:03:17,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:17,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:19,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:19,853 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:19,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:03:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:19,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:21,146 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:22,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:22,877 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:24,898 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:24,908 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:26,147 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:26,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:26,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:29,926 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:29,941 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:31,386 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:31,548 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:03:31,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:31,974 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:34,716 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:03:34,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:03:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:34,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:34,927 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:35,049 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:35,061 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:37,069 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:37,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:37,091 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:39,114 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:39,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:42,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:42,189 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:42,989 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:43,552 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:03:44,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:44,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:46,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:46,268 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:47,990 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:49,777 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:03:49,819 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:49,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:49,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:03:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:49,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:49,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:52,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:52,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:53,589 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:55,015 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:55,026 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:55,556 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:03:57,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:57,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:59,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:00,051 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:00,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:02,100 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:02,110 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:04,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:04,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:04,516 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:04,718 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:04:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:04:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:04,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:07,247 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:07,258 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:09,278 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:09,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:09,560 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:04:10,144 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:12,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:12,356 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:14,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:14,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:15,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:16,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:16,445 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:19,396 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:19,407 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:04:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:19,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:21,110 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:21,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:21,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:21,564 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:04:23,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:23,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:26,112 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:27,018 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:04:27,045 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:27,060 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:30,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:30,035 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:31,113 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:32,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:32,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:33,568 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:04:34,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:34,099 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:34,720 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:04:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:04:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:34,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:34,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:36,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:36,139 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:37,092 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:39,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:39,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:41,151 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:41,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:42,093 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:44,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:44,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:45,573 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:04:46,291 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:46,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:47,848 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:49,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:49,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:04:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:49,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:51,408 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:51,418 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:53,133 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:53,444 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:53,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:56,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:56,493 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:58,424 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:58,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:58,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:59,578 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:05:00,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:00,566 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:03,424 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:04,178 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:05:04,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:04,216 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:04,721 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:05:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:05:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:04,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:07,231 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:07,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:09,081 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:09,260 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:09,272 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:11,312 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:11,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:11,582 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:05:14,117 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:14,133 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:14,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:16,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:16,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:19,235 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:19,270 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:19,861 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:05:19,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:19,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:21,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:21,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:23,586 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:05:24,458 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:24,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:25,556 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:26,517 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:26,541 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:29,678 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:29,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:30,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:31,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:31,780 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:34,723 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:05:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:05:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:34,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:34,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:34,948 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:35,590 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:05:36,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:37,010 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:37,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:40,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:40,184 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:41,123 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:43,897 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:05:43,921 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:43,932 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:45,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:45,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:46,124 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:48,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:48,928 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:49,596 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:05:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:05:49,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:49,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:50,951 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:50,967 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:51,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:54,095 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:54,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:56,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:56,152 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:56,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:58,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:58,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:01,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:01,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:01,600 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:01,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:03,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:03,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:04,724 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:06:04,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:06:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:04,928 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:05,349 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:05,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:07,602 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:07,855 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:08,504 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:08,538 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:10,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:10,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:13,571 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:13,604 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:13,881 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:14,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:16,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:16,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:19,275 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:19,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:19,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:19,606 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:06:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:19,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:23,049 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:06:23,067 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:23,078 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:24,985 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:25,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:25,608 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:26,279 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:28,298 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:28,313 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:30,333 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:30,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:30,674 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:31,615 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:33,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:33,560 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:34,926 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:06:35,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:35,010 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:35,011 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:06:35,012 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:35,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:35,690 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:36,373 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:37,616 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:37,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:38,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:40,925 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:40,939 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:41,375 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:42,965 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:42,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:45,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:45,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:46,409 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:48,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:48,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:48,621 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:06:49,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:06:49,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:49,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:50,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:50,228 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:52,114 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:52,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:52,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:55,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:55,334 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:57,115 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:57,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:57,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:00,625 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:07:01,100 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:07:02,310 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:02,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:02,610 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:04,347 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:04,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:04,727 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:07:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:07:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:04,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:04,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:06,377 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:06,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:08,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:09,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:09,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:11,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:11,530 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:13,553 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:13,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:13,846 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:14,630 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:07:16,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:16,751 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:18,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:18,835 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:18,847 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:07:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:19,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:19,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:20,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:20,915 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:23,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:24,011 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:24,071 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:26,057 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:26,071 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:26,634 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:07:28,127 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:28,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:29,193 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:31,218 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:31,241 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:33,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:33,362 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:34,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:34,728 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:07:34,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:07:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:34,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:34,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:36,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:36,516 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:39,220 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:40,294 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:07:40,315 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:40,326 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:40,639 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:07:42,354 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:42,369 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:44,221 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:45,353 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:45,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:47,393 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:47,411 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:49,221 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:49,432 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:49,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:49,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:07:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:49,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:49,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:51,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:51,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:54,358 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:54,643 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:07:54,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:54,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:56,795 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:56,813 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:58,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:58,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:59,359 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:01,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:01,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:03,995 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:04,009 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:04,359 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:04,730 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:08:04,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:08:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:08:04,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:08:04,929 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:08:06,648 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:08:07,079 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:07,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:09,145 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:09,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:10,150 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:12,176 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:12,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:14,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:14,265 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:15,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:17,968 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:08:17,991 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:18,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:18,652 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:08:19,927 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:08:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:08:19,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:08:19,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:08:20,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:20,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:21,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:22,998 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:23,008 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:25,030 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:25,042 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:26,126 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:27,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:27,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:30,138 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:30,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:32,070 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:32,173 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:32,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:32,656 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:08:34,213 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:34,227 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:34,732 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:08:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:08:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:08:34,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:08:34,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:08:37,158 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:37,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:37,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:39,277 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:39,287 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:41,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:41,322 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:42,159 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:44,341 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:44,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:46,380 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:46,394 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:46,661 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:08:47,417 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:48,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:48,427 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:08:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:08:49,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:08:49,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:08:51,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:51,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:53,096 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:55,273 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:08:55,317 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:55,337 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:57,357 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:57,367 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:58,665 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:08:58,772 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:59,630 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:59,653 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:02,884 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:02,916 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:03,773 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:04,733 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:09:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:09:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:09:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:09:04,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:09:04,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:04,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:07,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:07,025 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:09,161 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:10,032 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:10,046 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:12,065 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:12,081 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:12,669 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:09:14,985 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:15,290 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:15,308 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:17,329 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:17,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:19,363 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:19,374 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:09:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:09:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:09:19,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:09:20,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:22,344 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:22,360 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:24,382 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:24,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:25,255 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:26,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:26,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:26,674 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:09:29,505 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:29,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:30,256 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:33,221 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:09:33,242 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:33,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:34,735 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:09:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:09:34,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:09:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:09:34,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:09:35,280 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:35,295 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:36,065 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:38,471 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:38,492 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:38,677 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:09:40,513 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:40,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:41,542 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:42,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:42,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:45,679 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:45,696 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:46,543 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:47,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:47,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:49,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:49,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:49,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:09:49,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:09:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:09:49,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:09:52,460 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:52,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:52,682 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:09:52,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:54,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:54,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:56,761 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:56,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:57,461 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:09:59,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:09:59,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:01,827 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:01,839 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:02,462 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:03,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:03,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:04,687 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:10:04,736 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:10:04,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:10:04,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:10:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:10:04,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:10:06,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:06,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:08,120 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:10,727 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:10:10,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:10,763 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:12,786 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:12,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:13,121 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:15,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:15,945 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:18,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:18,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:18,632 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:18,694 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:10:19,928 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:10:19,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:10:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:10:19,930 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:10:21,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:21,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:23,115 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:23,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:24,142 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:26,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:26,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:28,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:28,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:29,143 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:30,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:30,237 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:32,699 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:10:33,185 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:33,200 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:34,738 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:10:34,740 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:34,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:10:34,929 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:10:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:10:34,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:10:35,220 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:35,230 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:37,252 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:37,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:40,079 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:40,150 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:40,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:42,209 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:42,222 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:44,264 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:44,276 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:44,702 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:10:45,085 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:47,779 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:10:47,808 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:47,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:10:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:10:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:10:49,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:10:50,136 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:50,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:50,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:52,745 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:52,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:54,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:54,804 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:55,136 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:10:57,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:57,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:58,707 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:10:59,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:10:59,697 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:00,463 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:01,737 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:01,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:04,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:04,635 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:04,739 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:11:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:11:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:11:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:11:04,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:11:06,196 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:06,675 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:06,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:08,726 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:08,738 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:10,711 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:11:11,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:11,740 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:11,770 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:13,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:13,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:15,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:15,854 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:16,770 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:18,809 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:18,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:11:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:11:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:11:19,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:11:20,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:20,875 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:22,150 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:24,518 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:11:24,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:24,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:24,715 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:11:27,401 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:27,416 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:27,960 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:29,439 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:29,449 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:32,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:32,529 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:32,961 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:34,548 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:34,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:34,741 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:11:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:11:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:11:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:11:34,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:11:36,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:36,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:38,196 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:38,720 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:11:39,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:39,448 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:41,467 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:41,480 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:43,196 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:44,575 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:44,585 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:46,606 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:46,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:48,197 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:49,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:49,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:11:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:11:49,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:11:49,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:11:50,725 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:11:51,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:51,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:53,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:53,821 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:53,833 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:56,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:56,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:58,369 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:11:58,711 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:11:58,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:02,393 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:12:02,413 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:02,424 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:04,067 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:04,729 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:12:04,743 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:12:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:12:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:12:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:12:04,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:12:05,484 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:05,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:07,519 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:07,540 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:09,143 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:09,559 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:09,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:12,578 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:12,596 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:14,144 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:14,645 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:14,663 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:16,688 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:16,699 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:18,733 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:12:19,287 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:19,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:19,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:12:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:12:19,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:12:19,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:12:21,778 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:21,802 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:24,887 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:24,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:25,142 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:26,929 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:26,943 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:28,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:28,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:30,624 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:30,738 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:12:32,083 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:32,098 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:34,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:34,128 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:34,744 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:12:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:12:34,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:12:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:12:34,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:12:36,133 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:36,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:36,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:39,586 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:12:39,607 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:39,619 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:41,134 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:42,620 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:42,632 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:44,662 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:44,672 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:44,741 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:12:46,693 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:46,706 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:46,973 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:48,729 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:48,741 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:49,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:12:49,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:12:49,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:12:49,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:12:51,728 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:51,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:52,144 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:53,759 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:53,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:55,796 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:55,818 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:56,744 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:12:57,193 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:12:58,868 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:12:58,894 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:00,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:00,937 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:02,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:02,961 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:02,977 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:04,746 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:13:04,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:13:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:13:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:13:04,931 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:13:05,825 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:05,836 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:07,860 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:07,870 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:08,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:09,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:09,913 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:10,751 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:13:13,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:13,212 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:13,399 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:16,894 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:13:16,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:16,927 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:18,399 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:18,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:18,973 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:19,930 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:13:19,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:13:19,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:13:19,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:13:22,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:22,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:23,601 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:24,088 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:24,102 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:24,756 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:13:26,124 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:26,140 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:28,602 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:29,167 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:29,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:31,196 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:31,210 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:33,240 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:33,255 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:33,602 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:34,747 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:13:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:13:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:13:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:13:34,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:13:36,254 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:36,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:36,760 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:13:38,309 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:38,319 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:38,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:40,358 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:40,373 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:43,243 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:43,256 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:43,798 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:45,292 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:45,306 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:47,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:47,389 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:48,799 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:49,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:13:49,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:13:49,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:13:49,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:13:50,376 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:50,421 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:50,765 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:13:54,083 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:13:54,113 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:54,123 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:54,125 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:56,146 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:56,159 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:59,123 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:13:59,130 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:13:59,143 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:01,172 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:01,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:02,769 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:14:03,207 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:03,223 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:04,194 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:04,749 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:14:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:14:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:14:04,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:14:04,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:14:06,303 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:06,321 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:08,345 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:08,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:10,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:10,378 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:10,390 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:13,469 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:13,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:15,126 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:15,510 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:15,531 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:16,773 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:14:17,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:17,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:19,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:14:19,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:14:19,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:14:19,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:14:20,584 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:20,602 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:21,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:22,625 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:22,642 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:25,777 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:25,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:26,126 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:27,826 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:27,837 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:30,777 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:14:31,699 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:14:31,724 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:31,739 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:31,774 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:34,454 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:34,476 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:34,751 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:14:34,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:14:34,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:14:34,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:14:34,932 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:14:36,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:36,511 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:37,148 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:39,495 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:39,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:41,571 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:41,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:42,781 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:14:42,802 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:43,628 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:43,640 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:46,300 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:46,323 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:47,803 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:48,348 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:48,370 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:49,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:14:49,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:14:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:14:49,933 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:14:51,473 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:51,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:53,006 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:14:53,526 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:53,552 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:56,465 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:56,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:56,786 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:14:58,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:58,569 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:14:58,981 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:00,592 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:00,604 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:03,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:03,479 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:03,982 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:04,753 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:15:04,931 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:15:04,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:15:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:15:04,933 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:15:05,498 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:05,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:09,513 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:09,755 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:15:10,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:10,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:10,791 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:15:12,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:12,807 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:14,514 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:15,748 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:15,762 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:17,790 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:17,820 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:19,515 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:19,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:15:19,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:15:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:15:19,933 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:15:20,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:20,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:22,795 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:15:22,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:22,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:24,924 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:24,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:24,938 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:27,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:27,812 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:29,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:29,842 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:29,925 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:31,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:33,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:34,755 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:15:34,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:15:34,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:15:34,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:15:34,933 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:15:35,020 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:35,033 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:35,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:36,801 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:15:38,205 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:38,234 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:40,151 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:40,257 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:40,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:43,274 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:43,293 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:45,152 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:45,332 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:45,343 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:48,805 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:15:48,945 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:15:48,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:48,975 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:49,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:15:49,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:15:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:15:49,933 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:15:50,702 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:51,911 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:51,923 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:53,950 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:53,960 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:55,703 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:15:56,783 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:56,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:58,824 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:15:58,834 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:00,703 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:00,855 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:00,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:02,811 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:16:02,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:02,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:04,756 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:16:04,932 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:16:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:16:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:16:04,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:16:05,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:05,610 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:06,141 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:07,655 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:07,667 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:10,430 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:10,450 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:11,142 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:12,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:12,497 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:14,815 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:16:15,289 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:15,304 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:17,035 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:17,346 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:17,355 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:16:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:16:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:16:19,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:16:20,262 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:20,281 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:22,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:22,307 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:22,324 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:25,615 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:16:25,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:25,649 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:27,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:28,500 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:28,514 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:28,819 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:16:30,534 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:30,544 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:32,864 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:33,385 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:33,398 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:34,758 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:16:34,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:16:35,269 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:16:35,269 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:16:35,270 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:16:35,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:35,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:38,372 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:38,482 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:38,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:40,823 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:16:41,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:41,639 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:43,557 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:43,660 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:43,829 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:46,636 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:46,654 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:48,558 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:48,681 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:48,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:16:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:16:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:16:49,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:16:51,686 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:51,709 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:53,713 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:53,754 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:53,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:54,828 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:16:56,734 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:56,755 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:58,714 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:16:58,788 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:16:58,798 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:01,743 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:02,174 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:03,715 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:04,759 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:17:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:17:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:17:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:17:04,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:17:05,614 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:17:05,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:05,668 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:06,832 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:17:08,522 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:08,536 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:09,647 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:10,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:10,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:12,587 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:12,597 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:14,647 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:15,475 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:15,488 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:17,546 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:17,558 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:17:20,238 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:17:20,238 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:17:20,239 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:17:20,613 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:20,627 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:20,836 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:17:20,958 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:22,646 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:22,665 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:24,687 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:24,702 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:25,959 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:27,368 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:27,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:29,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:29,415 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:30,960 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:31,441 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:31,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:32,841 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:17:34,371 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:34,383 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:34,761 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:17:34,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:17:34,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:17:34,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:17:34,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:17:36,076 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:36,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:36,419 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:39,399 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:39,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:41,077 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:43,163 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:17:43,183 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:43,197 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:45,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:45,226 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:46,804 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:46,846 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:17:48,123 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:48,132 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:49,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:17:49,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:17:49,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:17:49,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:17:50,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:50,165 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:52,187 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:52,202 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:52,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:55,039 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:55,054 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:57,073 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:57,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:17:57,389 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:17:59,850 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:18:00,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:00,038 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:02,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:02,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:03,018 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:04,096 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:04,112 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:04,844 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:18:04,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:18:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:18:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:18:04,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:18:06,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:06,935 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:08,159 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:08,970 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:10,119 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:12,155 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:12,437 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:13,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:13,855 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:18:15,486 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:15,707 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:18,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:18,815 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:18,878 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:19,933 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:18:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:18:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:18:19,934 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:18:22,682 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:18:24,122 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:24,160 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:24,192 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:25,858 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:18:26,250 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:26,271 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:29,217 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:29,428 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:29,709 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:32,442 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:32,509 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:34,542 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:34,568 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:34,710 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:35,108 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:18:35,108 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:18:35,109 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:18:35,109 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:18:35,110 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:18:36,593 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:37,859 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:39,779 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:39,863 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:18:39,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:39,919 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:42,673 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:42,685 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:44,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:44,719 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:44,779 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:47,879 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:47,893 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:49,780 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:49,914 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:50,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:18:50,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:18:50,037 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:18:50,038 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:18:50,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:51,865 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:18:53,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:53,066 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:54,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:18:55,106 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:55,137 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:58,211 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:58,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:18:59,809 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:00,267 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:00,286 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:03,597 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:19:03,621 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:03,634 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:04,766 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:19:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:19:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:19:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:19:04,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:19:05,167 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:05,868 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:19:06,434 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:06,446 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:08,472 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:08,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:10,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:10,507 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:10,518 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:13,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:13,431 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:15,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:15,452 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:15,463 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:17,871 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:19:18,549 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:18,562 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:19:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:19:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:19:19,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:19:20,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:20,601 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:20,988 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:22,624 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:22,644 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:25,386 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:25,400 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:25,989 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:27,420 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:27,429 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:29,451 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:29,470 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:31,200 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:31,873 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:19:32,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:32,435 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:34,453 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:34,462 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:34,767 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:19:34,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:19:34,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:19:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:19:34,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:19:37,140 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:37,375 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:37,387 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:41,108 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:19:41,131 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:41,147 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:42,141 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:43,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:43,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:45,876 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:19:45,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:45,922 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:47,301 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:47,959 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:47,971 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:49,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:19:49,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:19:49,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:19:49,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:19:50,901 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:50,912 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:52,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:52,969 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:53,066 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:19:56,031 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:56,047 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:57,879 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:19:58,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:58,080 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:19:58,470 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:00,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:00,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:02,952 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:02,964 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:03,471 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:04,769 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:20:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:20:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:20:04,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:20:04,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:20:05,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:05,801 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:07,831 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:07,843 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:09,086 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:10,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:10,701 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:11,882 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:20:12,721 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:12,730 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:14,482 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:15,545 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:15,555 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:19,121 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:20:19,142 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:19,156 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:19,482 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:19,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:20:19,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:20:19,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:20:19,935 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:20:22,163 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:22,177 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:23,885 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:20:24,199 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:24,208 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:24,894 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:27,012 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:27,022 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:29,040 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:29,053 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:29,895 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:31,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:31,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:33,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:33,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:34,770 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:20:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:20:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:20:34,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:20:34,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:20:35,132 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:35,775 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:35,791 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:37,889 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:20:38,844 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:38,858 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:40,146 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:40,876 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:40,885 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:43,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:43,731 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:45,147 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:45,750 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:45,765 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:48,590 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:48,600 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:49,893 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:20:49,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:20:49,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:20:49,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:20:49,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:20:50,623 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:50,633 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:51,060 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:53,576 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:53,586 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:56,061 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:20:57,139 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:20:57,157 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:20:57,166 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:00,044 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:00,064 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:01,062 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:02,084 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:02,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:03,897 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:04,772 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:21:04,934 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:21:04,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:21:04,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:21:04,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:21:05,043 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:05,052 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:06,145 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:07,082 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:07,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:09,865 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:09,874 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:11,146 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:11,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:11,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:14,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:14,768 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:15,902 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:16,789 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:16,800 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:16,942 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:19,725 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:19,746 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:19,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:21:19,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:21:19,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:21:19,937 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:21:21,773 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:21,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:22,167 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:24,561 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:24,579 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:26,622 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:26,631 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:27,168 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:29,422 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:29,443 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:29,906 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:31,466 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:31,483 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:33,130 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:34,774 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:21:34,873 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:21:34,896 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:34,907 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:21:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:21:34,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:21:34,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:21:36,912 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 18:21:36,915 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 18:21:36,915 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:36,915 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:37,647 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:37,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:37,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:37,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 18:21:38,909 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:38,916 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:39,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:41,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:43,917 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:44,409 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:44,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:46,455 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:47,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:47,912 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:49,770 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:49,787 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:49,813 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:49,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:21:49,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:21:49,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:21:49,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:21:52,806 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:52,828 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:54,821 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:21:54,849 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:54,867 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:55,741 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 18:21:55,742 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,743 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,743 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,743 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,743 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,744 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,744 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,744 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,744 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,744 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,744 DEBUG SenderThread:1483490 [sender.py:send():382] send: history -2024-03-27 18:21:55,744 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:21:55,745 INFO SenderThread:1483490 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:21:55,915 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/wandb-summary.json -2024-03-27 18:21:56,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:57,916 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:21:57,918 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:57,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:59,947 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:21:59,957 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:00,746 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:01,976 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:01,985 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:04,776 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:22:04,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:04,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:22:04,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:22:04,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:22:04,936 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:22:04,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:06,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:06,962 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:06,972 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:09,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:09,013 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:11,072 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:11,903 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:12,186 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:15,720 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:22:15,744 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:15,758 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:16,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:18,532 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:18,543 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:19,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:22:20,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:22:20,053 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:22:20,054 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:22:20,563 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:21,212 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:21,817 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:23,848 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:23,863 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:26,213 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:26,708 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:26,958 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:29,003 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:29,072 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:31,214 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:31,905 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:32,158 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:34,188 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:34,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:22:35,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:22:35,125 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:22:35,125 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:22:35,127 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:22:35,359 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:36,329 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:37,379 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:37,414 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:39,433 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:40,589 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:41,330 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:42,608 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:42,864 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:45,703 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:45,936 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:46,331 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:47,990 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:48,005 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:49,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:22:49,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:22:49,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:22:49,937 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:22:50,742 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:50,752 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:51,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:52,772 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:52,785 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:56,635 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:22:56,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:56,692 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:56,727 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:22:58,713 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:22:58,722 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:01,638 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:01,657 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:01,728 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:03,676 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:03,694 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:04,779 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:23:04,935 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:23:04,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:23:04,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:23:04,937 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:23:05,720 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:06,760 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:07,163 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:08,781 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:08,794 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:11,756 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:11,779 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:12,165 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:13,797 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:13,811 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:16,889 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:16,902 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:17,165 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:18,930 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:18,942 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:19,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:23:19,936 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:23:19,937 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:23:19,937 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:23:22,027 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:22,041 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:23,073 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:24,059 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:24,069 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:27,055 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:27,068 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:28,642 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:29,093 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:29,103 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:29,948 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:23:32,239 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:32,251 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:33,642 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:34,818 DEBUG SenderThread:1483490 [sender.py:send():382] send: stats -2024-03-27 18:23:35,760 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: keepalive -2024-03-27 18:23:35,761 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:23:35,762 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:23:35,762 DEBUG SenderThread:1483490 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:23:35,968 ERROR gpu :1483490 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:23:35,989 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:36,002 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:37,951 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:23:38,045 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:38,062 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:38,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:41,109 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:41,134 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:43,154 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:43,171 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:43,950 DEBUG HandlerThread:1483490 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:23:45,953 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log -2024-03-27 18:23:46,296 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:46,318 ERROR gpu :1483490 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:23:47,954 INFO Thread-13 :1483490 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_141033-golaq7b9/files/output.log +2024-03-27 19:05:13,649 INFO StreamThr :1895319 [internal.py:wandb_internal():86] W&B internal server running at pid: 1895319, started at: 2024-03-27 19:05:13.648580 +2024-03-27 19:05:13,651 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status +2024-03-27 19:05:13,653 INFO WriterThread:1895319 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/run-7p2x8a0l.wandb +2024-03-27 19:05:13,654 DEBUG SenderThread:1895319 [sender.py:send():382] send: header +2024-03-27 19:05:13,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: run +2024-03-27 19:05:14,021 INFO SenderThread:1895319 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files +2024-03-27 19:05:14,021 INFO SenderThread:1895319 [sender.py:_start_run_threads():1136] run started: 7p2x8a0l with start time 1711562713.647808 +2024-03-27 19:05:14,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 19:05:14,029 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: check_version +2024-03-27 19:05:14,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 19:05:14,643 DEBUG HandlerThread:1895319 [system_info.py:__init__():27] System info init +2024-03-27 19:05:14,643 DEBUG HandlerThread:1895319 [system_info.py:__init__():42] System info init done +2024-03-27 19:05:14,643 INFO HandlerThread:1895319 [system_monitor.py:start():194] Starting system monitor +2024-03-27 19:05:14,643 INFO SystemMonitor:1895319 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 19:05:14,644 INFO HandlerThread:1895319 [system_monitor.py:probe():214] Collecting system info +2024-03-27 19:05:14,645 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 19:05:14,646 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started disk monitoring +2024-03-27 19:05:14,647 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 19:05:14,648 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started memory monitoring +2024-03-27 19:05:14,650 INFO SystemMonitor:1895319 [interfaces.py:start():190] Started network monitoring +2024-03-27 19:05:14,699 DEBUG HandlerThread:1895319 [system_info.py:probe():151] Probing system +2024-03-27 19:05:14,705 DEBUG HandlerThread:1895319 [system_info.py:_probe_git():136] Probing git +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_info.py:_probe_git():144] Probing git done +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_info.py:probe():199] Probing system done +2024-03-27 19:05:14,716 DEBUG HandlerThread:1895319 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T18:05:14.699269', 'startedAt': '2024-03-27T18:05:13.643873', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--save_total_limit=1', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--freeze_encoder', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft-frozen-encoder', 'commit': 'e7946df277d73ac75c34c2017b01c6d39e0275cd'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft-frozen-encoder', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2055.302406250001, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 1785.051, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.027, 'min': 1500.0, 'max': 2250.0}, {'current': 1716.844, 'min': 1500.0, 'max': 2250.0}, {'current': 2279.976, 'min': 1500.0, 'max': 2250.0}, {'current': 3328.296, 'min': 1500.0, 'max': 2250.0}, {'current': 2778.772, 'min': 1500.0, 'max': 2250.0}, {'current': 1670.514, 'min': 1500.0, 'max': 2250.0}, {'current': 1679.17, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.11, 'min': 1500.0, 'max': 2250.0}, {'current': 3342.154, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.65, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.492, 'min': 1500.0, 'max': 2250.0}, {'current': 3344.267, 'min': 1500.0, 'max': 2250.0}, {'current': 1991.129, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.832, 'min': 1500.0, 'max': 2250.0}, {'current': 1681.287, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.975, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.691, 'min': 1500.0, 'max': 2250.0}, {'current': 3343.923, 'min': 1500.0, 'max': 2250.0}, {'current': 1714.613, 'min': 1500.0, 'max': 2250.0}, {'current': 1711.757, 'min': 1500.0, 'max': 2250.0}, {'current': 2605.829, 'min': 1500.0, 'max': 2250.0}, {'current': 1715.095, 'min': 1500.0, 'max': 2250.0}, {'current': 1868.607, 'min': 1500.0, 'max': 2250.0}, {'current': 1761.225, 'min': 1500.0, 'max': 2250.0}, {'current': 1722.608, 'min': 1500.0, 'max': 2250.0}, {'current': 2446.611, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.419, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.485, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.608, 'min': 1500.0, 'max': 2250.0}, {'current': 1774.005, 'min': 1500.0, 'max': 2250.0}, {'current': 3318.879, 'min': 1500.0, 'max': 2250.0}, {'current': 3356.822, 'min': 1500.0, 'max': 2250.0}, {'current': 1674.163, 'min': 1500.0, 'max': 2250.0}, {'current': 1676.528, 'min': 1500.0, 'max': 2250.0}, {'current': 3346.209, 'min': 1500.0, 'max': 2250.0}, {'current': 1679.409, 'min': 1500.0, 'max': 2250.0}, {'current': 1673.8, 'min': 1500.0, 'max': 2250.0}, {'current': 1675.44, 'min': 1500.0, 'max': 2250.0}, {'current': 1722.367, 'min': 1500.0, 'max': 2250.0}, {'current': 1721.361, 'min': 1500.0, 'max': 2250.0}, {'current': 1719.425, 'min': 1500.0, 'max': 2250.0}, {'current': 2130.252, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.838, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.535, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.69, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.257, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.417, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.058, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.79, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.254, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.759, 'min': 1500.0, 'max': 2250.0}, {'current': 3285.802, 'min': 1500.0, 'max': 2250.0}, {'current': 3327.549, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.454, 'min': 1500.0, 'max': 2250.0}, {'current': 2291.123, 'min': 1500.0, 'max': 2250.0}, {'current': 1845.287, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.335, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.798, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.761, 'min': 1500.0, 'max': 2250.0}, {'current': 2172.943, 'min': 1500.0, 'max': 2250.0}, {'current': 2109.384, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.332, 'min': 1500.0, 'max': 2250.0}, {'current': 2173.511, 'min': 1500.0, 'max': 2250.0}, {'current': 2187.364, 'min': 1500.0, 'max': 2250.0}, {'current': 2152.119, 'min': 1500.0, 'max': 2250.0}, {'current': 3315.314, 'min': 1500.0, 'max': 2250.0}, {'current': 3333.811, 'min': 1500.0, 'max': 2250.0}, {'current': 1966.133, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.353, 'min': 1500.0, 'max': 2250.0}, {'current': 2183.755, 'min': 1500.0, 'max': 2250.0}, {'current': 1836.471, 'min': 1500.0, 'max': 2250.0}, {'current': 3319.659, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.1, 'min': 1500.0, 'max': 2250.0}, {'current': 3156.296, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.519, 'min': 1500.0, 'max': 2250.0}, {'current': 3259.492, 'min': 1500.0, 'max': 2250.0}, {'current': 2267.628, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.441, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.329, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.974, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.269, 'min': 1500.0, 'max': 2250.0}, {'current': 3315.801, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.948, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.171, 'min': 1500.0, 'max': 2250.0}, {'current': 3147.217, 'min': 1500.0, 'max': 2250.0}, {'current': 1663.184, 'min': 1500.0, 'max': 2250.0}, {'current': 2145.9, 'min': 1500.0, 'max': 2250.0}, {'current': 2302.183, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1812.149, 'min': 1500.0, 'max': 2250.0}, {'current': 1739.416, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.942, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.725, 'min': 1500.0, 'max': 2250.0}, {'current': 1846.358, 'min': 1500.0, 'max': 2250.0}, {'current': 3324.686, 'min': 1500.0, 'max': 2250.0}, {'current': 3313.397, 'min': 1500.0, 'max': 2250.0}, {'current': 1664.908, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.332, 'min': 1500.0, 'max': 2250.0}, {'current': 3357.779, 'min': 1500.0, 'max': 2250.0}, {'current': 2196.764, 'min': 1500.0, 'max': 2250.0}, {'current': 1629.818, 'min': 1500.0, 'max': 2250.0}, {'current': 1548.482, 'min': 1500.0, 'max': 2250.0}, {'current': 1550.408, 'min': 1500.0, 'max': 2250.0}, {'current': 1551.062, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.314, 'min': 1500.0, 'max': 2250.0}, {'current': 2797.062, 'min': 1500.0, 'max': 2250.0}, {'current': 1556.743, 'min': 1500.0, 'max': 2250.0}, {'current': 2040.048, 'min': 1500.0, 'max': 2250.0}, {'current': 1557.951, 'min': 1500.0, 'max': 2250.0}, {'current': 1548.578, 'min': 1500.0, 'max': 2250.0}, {'current': 2204.931, 'min': 1500.0, 'max': 2250.0}, {'current': 2191.05, 'min': 1500.0, 'max': 2250.0}, {'current': 2206.083, 'min': 1500.0, 'max': 2250.0}, {'current': 2179.071, 'min': 1500.0, 'max': 2250.0}, {'current': 2137.078, 'min': 1500.0, 'max': 2250.0}, {'current': 2908.115, 'min': 1500.0, 'max': 2250.0}, {'current': 3329.978, 'min': 1500.0, 'max': 2250.0}, {'current': 2146.791, 'min': 1500.0, 'max': 2250.0}, {'current': 1835.104, 'min': 1500.0, 'max': 2250.0}, {'current': 3279.409, 'min': 1500.0, 'max': 2250.0}, {'current': 2340.495, 'min': 1500.0, 'max': 2250.0}, {'current': 2350.105, 'min': 1500.0, 'max': 2250.0}, {'current': 1860.834, 'min': 1500.0, 'max': 2250.0}, {'current': 1928.589, 'min': 1500.0, 'max': 2250.0}, {'current': 3299.491, 'min': 1500.0, 'max': 2250.0}, {'current': 1897.289, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1610.3510513305664}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 19:05:14,716 INFO HandlerThread:1895319 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 19:05:14,716 INFO HandlerThread:1895319 [system_monitor.py:probe():227] Publishing system info +2024-03-27 19:05:14,718 INFO HandlerThread:1895319 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 19:05:14,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: files +2024-03-27 19:05:14,722 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 19:05:14,726 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 19:05:14,727 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: python_packages +2024-03-27 19:05:14,728 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:14,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:14,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:14,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,960 DEBUG SenderThread:1895319 [sender.py:send():382] send: config +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:05:14,961 WARNING SenderThread:1895319 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 19:05:14,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: telemetry +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/requirements.txt +2024-03-27 19:05:15,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-metadata.json +2024-03-27 19:05:15,268 INFO wandb-upload_0:1895319 [upload_job.py:push():131] Uploaded file /tmp/tmpdusptjiswandb/d02ankfh-wandb-metadata.json +2024-03-27 19:05:16,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:17,022 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:18,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:19,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:19,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,793 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:23,963 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:24,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:24,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,752 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,026 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:29,424 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:29,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:31,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:32,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:32,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:35,001 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:35,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:35,028 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:35,074 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:37,029 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:37,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:37,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:39,029 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:40,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,405 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:41,030 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:42,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:42,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,030 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:45,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:45,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:45,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:46,250 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:47,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:47,032 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:05:47,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:47,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:49,032 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:50,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:50,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:51,033 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:51,857 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:53,033 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:53,896 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:05:53,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:53,931 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,034 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:55,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:57,035 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:05:57,850 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:58,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:58,826 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:59,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:59,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:00,036 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:00,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:02,036 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:03,461 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:03,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:03,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:04,037 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:05,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:05,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:06,038 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:08,038 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:08,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:09,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:10,039 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:10,114 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:10,116 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:06:10,117 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:10,117 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:11,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:11,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:11,039 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:06:12,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:13,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:13,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:13,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:14,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:14,426 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:14,651 DEBUG SystemMonitor:1895319 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 19:06:14,653 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:06:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:14,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:15,938 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:15,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:16,041 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:18,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:18,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:18,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:20,040 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:20,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:20,993 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:21,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:21,042 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:06:22,043 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:23,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:23,892 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:24,043 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:25,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:25,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:26,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:26,044 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:28,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:28,922 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:28,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:29,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:30,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:31,652 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:32,046 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:32,505 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:32,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,537 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:34,047 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:35,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:35,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:36,047 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:37,625 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:37,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:38,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:40,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:40,649 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:40,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,049 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:42,699 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,708 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:43,246 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:44,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:44,655 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:06:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:44,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:45,552 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:45,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:46,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:47,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:48,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:49,216 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:50,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:50,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:50,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:51,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:51,556 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:06:51,557 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:51,557 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:52,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:06:52,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:52,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:52,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:53,052 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:54,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:54,301 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:55,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:55,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:56,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:57,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:57,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:58,054 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:06:59,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:59,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:59,923 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:00,054 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:00,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:00,814 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:02,055 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:02,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:02,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,055 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:05,869 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:06,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:07,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:07,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:08,057 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:10,057 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:11,497 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:11,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,526 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,539 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:12,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:14,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:14,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,532 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,657 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:07:14,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:14,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:16,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:16,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:16,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:17,498 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:18,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:19,608 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:20,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:21,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:21,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:22,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:23,113 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:24,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:24,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:24,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:26,546 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:28,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:29,072 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:29,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:29,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:29,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:29,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:30,063 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:31,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:31,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:32,063 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:33,067 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:07:33,069 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:07:33,070 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:07:33,070 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:07:34,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:07:34,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:34,587 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,724 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:35,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:36,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:36,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:36,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:38,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:39,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,671 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:40,066 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:40,699 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:41,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:41,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:42,067 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:44,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:44,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,658 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:07:44,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:44,728 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:45,861 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:46,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:46,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:46,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:48,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:50,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:50,459 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:51,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:51,299 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:51,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:52,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:53,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:53,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:54,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:55,360 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:56,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:57,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:58,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:07:58,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:58,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:00,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:00,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,165 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:02,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:02,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:03,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:03,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:04,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:05,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:05,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:06,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:08,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:08,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,221 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,902 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:10,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:10,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:10,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:12,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:13,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:14,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:14,431 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:14,435 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:14,437 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:08:14,437 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:14,437 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:14,660 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:08:14,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:14,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:15,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:08:15,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:15,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:17,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:18,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:18,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:20,111 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:20,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:22,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:23,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,125 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:24,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:25,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:25,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:25,860 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:26,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:28,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:28,670 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:28,690 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:30,081 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:31,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,823 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:32,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:33,612 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:34,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:36,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:36,527 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:36,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:37,465 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:38,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:38,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,592 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:40,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:41,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:41,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:42,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:43,442 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:43,478 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:44,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:44,662 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:08:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:44,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:46,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:46,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:46,472 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:48,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:48,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:48,503 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:49,096 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:50,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:50,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:52,088 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:53,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:53,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,088 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:55,075 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:55,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:55,754 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:08:55,755 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:55,755 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:56,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:08:56,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:57,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:58,089 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:08:58,480 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,492 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:59,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:00,090 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:00,511 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:00,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:00,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:02,091 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:03,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,658 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:04,092 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:06,093 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:06,694 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:07,258 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:07,276 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:07,290 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,094 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:10,094 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:10,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:10,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,095 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:12,199 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,342 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:14,096 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:14,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:14,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:14,663 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:09:14,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:14,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:16,097 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:16,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:17,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:18,097 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:18,331 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:19,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:19,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:20,098 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:22,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,098 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:23,479 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:24,099 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:24,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:24,110 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:26,100 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:27,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:28,100 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:29,120 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:29,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:29,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:30,101 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:32,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,102 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:32,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,102 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:34,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:35,115 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:36,103 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:37,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,305 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,465 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:09:37,466 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:09:37,467 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:09:37,467 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:09:38,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:09:38,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:39,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:39,329 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:39,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:40,104 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:40,131 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:42,105 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:42,188 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:42,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:44,105 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:44,664 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:09:44,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:44,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:44,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:45,769 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:45,779 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:45,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:45,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:46,106 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:47,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:48,107 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:48,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:50,107 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:50,781 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:50,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:50,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,108 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:52,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:54,109 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:55,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:55,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,109 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:56,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:57,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:57,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:58,110 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:09:59,728 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:59,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:00,110 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:00,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:00,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:02,111 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:02,408 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:02,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:02,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:04,111 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:04,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:04,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:06,112 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:07,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:07,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,113 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:08,395 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:10,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,113 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:12,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,114 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:14,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:14,115 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:14,666 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:10:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:14,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:14,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:16,116 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:16,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:16,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,116 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:18,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,870 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:10:18,872 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:10:18,872 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:10:18,872 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:10:19,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:10:19,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:20,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:21,117 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:22,118 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:22,132 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:22,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:22,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:24,118 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:25,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,511 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:26,119 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:27,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:27,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:28,120 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:29,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:29,729 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:29,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:30,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,120 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:31,481 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:32,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:32,121 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:32,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,122 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:34,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:36,122 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:37,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:37,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:37,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:38,123 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:39,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:39,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,124 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:41,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:41,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,124 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:42,286 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:43,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:43,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:44,125 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:44,668 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:10:44,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:44,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:45,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:46,126 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:48,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,126 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:48,271 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:50,127 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:50,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:50,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,127 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:52,811 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:53,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:54,128 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:54,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,129 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:58,130 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:10:58,264 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:58,291 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:58,309 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:58,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:59,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:59,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:59,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:00,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:00,582 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:00,584 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:11:00,584 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:00,585 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:01,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:11:02,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:03,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:03,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:03,131 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:04,132 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:04,140 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:05,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:05,923 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:06,132 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:07,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:07,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:08,133 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:09,302 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:09,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:09,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:10,134 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:12,134 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:13,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:13,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:14,135 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:14,362 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:14,669 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:11:14,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:15,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:15,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,135 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:18,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:18,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:18,136 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:20,137 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:20,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:20,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:20,330 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:22,138 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:23,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:23,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:24,138 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:25,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:25,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:26,321 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:28,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:28,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:28,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:29,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:29,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:30,140 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:30,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:30,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:31,531 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:32,141 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:33,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:33,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:34,141 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:36,142 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:37,001 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:37,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:37,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:37,496 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:38,143 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:39,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:40,144 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:41,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:41,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:43,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:44,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,671 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:11:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:46,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:48,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:52,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:53,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:53,065 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:54,091 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:54,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:54,773 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:54,774 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:11:54,774 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:54,775 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:55,148 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:11:55,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:55,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:56,149 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:57,149 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:57,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:57,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:58,150 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:11:59,402 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:59,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:59,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:00,150 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:00,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:00,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:02,151 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:03,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:03,020 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:04,151 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:05,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:06,143 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,152 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:06,154 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,152 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:08,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:10,153 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:11,028 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:11,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:11,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:12,154 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:14,155 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:14,672 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:12:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:14,874 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:14,894 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:14,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,155 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:16,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:17,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:17,916 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:18,157 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:19,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:19,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:20,157 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:22,158 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:22,366 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:22,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:22,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:24,158 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:25,004 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:25,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,159 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:27,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,015 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:28,160 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:29,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:29,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:30,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:30,037 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:30,160 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:32,161 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:33,037 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:34,006 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:34,161 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:35,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:35,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:36,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:36,327 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:12:36,329 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:12:36,329 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:12:36,330 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:12:37,125 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,142 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:12:38,162 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:39,163 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:39,640 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:40,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,163 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:42,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,165 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:44,165 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:44,674 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:12:44,677 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:44,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:44,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:44,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,930 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,166 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:46,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:47,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:48,167 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:49,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,939 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:50,120 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:50,167 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:52,168 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:53,525 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:53,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:53,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:54,169 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:55,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:55,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:56,108 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:56,169 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:58,170 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:12:58,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:58,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:59,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:59,731 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:59,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:00,171 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:00,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:00,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:01,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:02,172 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:03,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:03,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:04,173 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:05,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:05,434 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:06,174 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:07,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:08,174 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:08,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:08,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,175 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:10,569 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:12,176 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:13,407 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:13,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:13,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:14,176 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:14,675 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:13:14,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:14,730 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:15,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:16,177 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:17,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:18,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:18,388 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:18,390 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:13:18,390 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:18,391 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:19,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:13:19,391 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:20,178 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:20,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:20,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:21,179 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:22,179 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:22,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,537 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:24,180 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:25,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:25,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:25,434 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:26,181 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:27,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:28,181 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:29,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:29,731 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:30,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:30,182 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:31,082 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:31,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:31,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,183 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:33,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:33,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:34,183 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:35,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:35,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:36,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:36,184 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:37,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:37,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:38,185 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:40,185 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:40,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:40,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:41,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:42,188 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:42,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:44,190 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:44,677 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:13:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:44,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:44,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:45,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:45,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:46,191 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:46,398 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:47,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:47,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:48,191 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:50,192 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:50,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:50,844 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:51,399 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:52,193 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:52,862 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:52,875 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,194 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:55,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:55,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:56,194 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:57,377 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:57,801 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:57,814 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:58,195 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:13:59,684 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:59,686 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:13:59,686 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:59,686 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:59,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:59,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:00,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:14:00,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:00,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:00,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:01,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:02,196 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:02,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:02,852 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:03,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:04,197 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:05,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:05,721 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:06,198 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:08,198 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:08,990 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:09,298 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:09,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:10,199 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:11,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:11,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:12,199 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:14,107 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,200 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:14,636 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:14,678 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:14:14,731 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:14,732 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:16,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:16,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:16,201 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:18,201 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:19,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:19,217 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:20,202 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:20,620 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:21,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:21,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:22,207 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:24,207 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:24,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,255 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,208 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:26,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:26,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:28,209 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:29,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:29,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:29,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:30,209 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:31,289 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:31,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:31,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:32,210 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:34,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,210 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:36,211 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:36,218 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:36,227 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:37,273 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:38,211 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:39,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:39,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,212 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:41,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:41,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:41,284 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:14:41,285 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:14:41,286 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:14:41,286 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:14:42,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:14:42,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:42,287 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:43,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:44,213 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:44,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:44,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:44,680 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:14:44,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:44,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:44,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:46,214 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:47,291 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:47,867 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:47,885 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:48,215 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:50,216 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:50,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:51,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:52,216 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:52,947 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:53,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:53,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:54,217 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:56,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,218 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:57,952 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:58,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,115 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,218 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:14:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:59,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:00,219 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:00,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:00,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:02,220 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:03,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:04,221 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:05,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:05,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:06,222 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:07,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:07,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:08,222 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:09,594 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:10,223 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:10,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:10,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:12,224 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:12,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:12,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:14,225 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:14,681 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:15:14,684 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:14,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:14,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:15,809 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:15,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,225 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:17,843 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:17,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:18,226 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:20,226 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:20,571 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:20,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:20,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,227 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:22,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:22,922 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:15:22,923 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:15:22,924 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:15:22,924 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:15:23,227 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:15:24,228 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:25,228 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:26,188 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:26,218 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:26,226 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:26,229 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:26,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,229 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:28,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:29,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:30,230 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:31,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:32,216 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:32,230 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:33,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:33,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:34,231 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:35,765 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:35,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:36,232 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:37,795 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,867 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:38,232 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:40,233 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:40,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:40,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:42,233 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:42,696 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:42,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:43,848 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:44,234 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:44,683 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:15:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:45,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:45,752 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:46,235 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:47,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:48,235 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:49,507 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:50,236 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:50,794 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:50,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,237 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:52,827 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:54,238 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:55,385 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:55,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:55,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:56,239 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:57,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:57,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:58,240 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:15:59,733 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:59,734 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:00,240 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:00,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:00,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:00,964 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:02,241 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:04,242 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:04,406 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:16:04,408 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:16:04,408 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:16:04,409 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:16:04,604 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:04,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,634 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:05,242 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:16:06,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:06,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:06,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:07,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:07,460 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:08,243 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:09,480 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:09,489 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:10,244 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:11,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:11,518 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:12,048 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:12,244 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:14,245 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:14,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,685 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:16:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:14,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:16,246 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:16,498 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:16,507 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:17,701 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:18,246 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:19,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:20,247 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:21,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:21,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:22,248 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:23,687 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:24,249 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:24,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:24,403 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,250 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:26,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:28,250 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:29,348 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:29,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:29,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:30,251 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:31,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:32,252 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:34,252 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:34,609 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:34,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:35,328 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:36,253 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:36,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:38,254 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:39,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:39,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:40,255 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:40,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:42,255 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:43,258 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:43,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:44,256 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:44,687 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:16:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:45,988 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:46,001 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:16:46,002 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:16:46,003 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:16:46,003 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:16:46,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,257 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:16:46,257 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:47,258 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:48,258 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:48,262 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:48,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:50,259 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:50,530 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,366 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:52,260 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:53,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:53,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:54,260 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:56,261 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:56,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:57,622 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:58,261 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:16:58,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:58,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:59,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:00,262 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:01,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:02,263 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:03,408 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:03,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:03,593 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:04,263 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:06,264 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:06,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,264 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:08,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:08,599 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:10,265 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:11,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:12,265 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:13,609 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:13,675 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:13,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:14,266 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:14,688 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:17:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:14,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:16,267 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:16,593 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:16,603 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:18,267 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:18,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:18,634 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:19,257 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:20,268 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:22,021 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:22,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:22,058 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:22,268 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:24,269 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:24,271 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:24,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:24,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,270 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:26,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,609 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:17:27,611 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:17:27,611 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:17:27,611 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:17:28,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:17:28,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:29,271 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:29,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:29,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:29,950 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:30,275 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:31,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:31,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:32,276 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:34,276 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:34,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:34,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:35,265 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:36,277 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:37,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:38,277 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:40,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,020 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,278 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:40,920 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:42,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:42,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:42,279 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:44,279 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:44,691 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:17:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:44,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:44,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:44,995 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:46,280 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:46,911 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:47,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:47,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,281 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:52,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:52,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:52,281 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:52,562 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:54,282 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:54,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:54,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:56,283 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:57,007 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:57,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:58,229 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:58,283 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:17:59,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:59,735 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:00,284 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:00,620 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:00,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:00,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,593 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:05,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:05,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,581 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:10,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,632 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,951 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:14,288 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:14,692 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:18:14,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:15,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,705 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:16,289 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:18,290 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:18,701 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:18,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:19,580 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:20,290 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:20,733 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:20,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:21,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:18:21,928 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:18:21,928 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:18:21,928 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:18:22,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:18:22,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:23,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:23,638 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:23,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:24,292 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:25,240 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:25,668 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:25,684 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:26,293 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:28,294 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:28,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:28,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:29,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:30,294 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:30,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:30,750 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:31,230 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:32,295 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:33,651 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:33,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,296 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:35,680 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:35,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:36,296 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:36,891 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:38,297 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:39,139 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:39,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:39,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:40,298 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:41,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,298 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:42,888 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:44,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,299 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:44,694 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:18:44,734 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:44,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:46,300 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:47,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,105 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:48,301 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:48,550 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:49,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:50,301 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:52,267 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,302 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:54,302 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:54,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,545 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:56,303 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:57,208 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:57,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:58,304 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:18:59,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:59,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:59,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:00,304 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:02,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,305 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:02,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:03,553 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:19:03,555 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:19:03,555 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:19:03,556 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:19:04,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:19:04,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:04,403 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:04,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:05,198 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:05,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:06,306 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:07,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:08,307 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:09,360 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:09,374 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,308 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:11,190 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:12,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,309 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:14,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:14,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:14,309 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:14,696 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:19:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:16,310 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:16,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:17,926 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:17,945 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:18,311 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:20,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:20,874 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:21,313 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:22,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:22,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:23,313 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:24,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:25,314 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:27,315 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:27,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:28,508 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:29,315 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:29,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:29,889 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:29,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,316 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:31,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:33,317 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:34,506 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:34,715 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:34,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:35,317 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:36,743 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:36,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:37,318 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:39,319 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:39,756 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:39,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,161 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:41,319 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:41,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:41,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,320 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:43,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:44,698 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:19:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:44,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:45,172 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:19:45,174 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:19:45,174 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:19:45,175 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:45,175 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:19:45,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:19:45,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:46,614 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:46,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,321 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:48,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:48,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:49,322 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:51,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:51,323 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:51,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:51,784 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:53,324 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:55,325 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:55,344 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:55,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:56,804 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:57,325 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:58,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:58,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:59,326 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:19:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:00,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:00,486 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:01,327 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:02,792 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:03,328 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:03,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:03,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:05,333 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:05,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:05,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:07,333 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:08,453 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:08,677 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:08,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:09,334 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:10,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:10,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:11,335 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:13,335 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:13,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:13,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:14,428 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:14,699 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:20:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:14,736 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:15,336 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:15,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:15,775 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,336 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:18,765 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:18,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:19,337 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:20,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:20,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:20,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:21,337 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:23,338 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:23,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:23,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,093 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:25,338 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:25,843 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:26,781 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:20:26,782 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:20:26,782 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:20:26,783 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:20:27,339 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:20:27,339 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:28,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:28,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:29,340 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:29,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:30,898 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:30,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:31,077 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:31,340 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:33,341 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:34,518 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:34,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:34,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:35,342 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:36,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:37,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,342 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:39,343 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:39,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:39,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,343 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:41,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,726 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:43,344 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:44,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:44,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:44,701 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:20:44,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:44,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:45,345 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:46,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:46,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:46,730 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:47,346 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:48,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:48,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:49,346 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:50,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:50,993 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:51,347 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:52,729 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:53,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:53,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:53,348 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:55,046 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:55,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:55,348 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:57,349 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:57,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:57,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,382 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:59,349 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:20:59,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:00,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:00,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:01,350 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:03,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,351 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:04,376 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:05,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:05,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:05,351 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:07,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,352 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:08,376 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:21:08,378 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:21:08,378 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:21:08,379 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:21:09,353 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:21:09,353 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:09,379 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:10,433 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:10,455 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:10,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:11,354 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:13,354 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:13,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,459 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:14,704 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:21:14,706 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:14,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:14,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:15,356 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:15,481 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:15,492 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:17,357 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:18,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,519 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:19,357 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:20,018 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:20,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:20,551 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:21,358 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:23,358 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:23,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,359 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:25,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,997 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:27,359 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:27,520 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:27,536 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:29,360 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:29,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:30,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:30,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:31,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:31,361 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:32,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:32,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:33,361 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:35,281 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,362 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:36,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:37,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,325 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,363 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:39,363 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:40,283 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,308 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:41,364 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:41,666 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:42,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:42,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:43,365 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:44,704 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:21:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:44,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:45,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,366 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:47,367 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:47,648 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:48,933 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:48,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:48,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:49,367 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:49,992 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:21:49,993 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:21:49,994 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:21:49,994 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:21:50,368 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:21:51,368 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:51,961 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:51,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:52,661 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:53,369 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:53,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:54,008 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,369 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:56,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:56,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,370 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:58,071 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:58,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:58,311 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:59,370 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:21:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:59,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:00,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:00,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:01,371 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:03,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,372 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:04,309 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:05,280 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,373 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:07,374 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:08,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:08,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:09,374 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:09,979 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:10,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:10,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,375 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:13,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,376 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:14,705 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:22:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:14,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:15,376 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:15,398 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:15,408 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:15,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:17,377 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:18,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:18,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:19,378 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:20,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:21,378 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:22,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:22,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:23,379 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:25,380 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:26,464 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:26,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:27,380 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:29,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:29,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:29,381 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:29,737 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:31,382 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:31,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,623 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:31,637 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:22:31,638 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:22:31,638 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:22:31,639 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:22:32,382 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:22:33,383 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:34,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:34,554 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:35,383 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:36,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,640 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:37,384 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:39,385 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:39,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:39,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,385 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:41,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,631 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,656 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:43,386 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:43,651 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:44,707 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:22:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:45,386 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:45,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:45,699 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:46,661 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:47,387 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:47,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:47,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:49,388 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:50,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:50,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,388 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:51,666 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:52,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:52,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:53,389 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:55,390 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:56,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,101 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,672 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:57,390 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:58,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:58,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:59,391 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:22:59,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:01,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,392 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:01,674 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:03,393 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:04,745 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:04,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:04,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:05,393 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:06,694 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:06,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:06,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:07,394 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:08,846 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,858 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:09,395 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:11,395 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:11,716 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:11,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:11,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:13,394 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:13,395 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:23:13,395 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:13,396 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:23:13,396 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:13,396 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:13,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:14,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:14,709 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:23:14,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:14,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:15,397 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:16,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,727 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:17,397 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:18,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:18,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:19,398 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:20,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:20,808 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:21,399 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:21,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:22,826 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:22,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:23,399 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:25,400 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:25,730 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:25,745 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:26,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:27,400 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:27,766 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,401 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:29,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:29,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,401 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:32,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:32,957 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:32,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:33,402 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:34,995 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:35,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:35,402 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:37,033 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,403 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:37,785 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:39,404 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:40,533 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:40,562 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:40,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:41,404 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:42,806 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:43,405 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:43,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:43,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:44,710 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:23:44,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:45,406 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:45,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:45,460 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:47,406 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:47,819 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:48,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:48,490 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:49,407 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:50,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:50,523 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:51,408 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:52,832 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:53,408 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:53,618 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:55,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:55,195 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:23:55,196 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:55,196 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:55,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:23:55,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:55,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:55,662 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,409 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:57,844 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:58,705 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:58,720 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:59,410 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:23:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:00,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:00,751 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:01,411 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:02,866 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:03,411 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:03,775 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:03,793 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,412 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:05,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,412 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:07,890 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:08,800 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:08,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:09,413 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:10,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:10,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:11,414 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:12,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:13,415 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:13,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:13,831 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:14,711 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:24:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:14,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:15,415 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:15,850 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:15,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,416 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:17,902 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:19,381 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:19,402 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,416 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:21,417 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:22,278 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:23,417 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:23,840 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:24,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:24,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,396 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:28,841 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:29,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:29,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:31,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:31,478 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:33,953 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:34,447 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:34,465 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,498 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:37,421 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:39,181 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:39,404 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,422 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:41,423 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:41,436 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:41,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:43,423 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:44,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,529 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:44,713 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:24:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:44,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:45,425 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:46,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:46,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,425 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:48,551 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:24:48,552 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:24:48,553 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:24:48,553 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:24:49,426 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:24:49,426 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:49,473 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,485 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,554 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:51,427 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:51,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:51,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:53,428 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:54,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:55,207 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:55,429 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:57,430 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:58,146 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:58,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:58,180 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,430 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:24:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:59,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:00,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:00,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:00,218 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:01,431 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:03,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,432 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:05,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,220 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:05,432 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:07,433 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:08,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:08,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:09,434 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:10,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:11,434 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:13,095 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,435 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:14,715 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:25:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:15,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,162 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,436 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:16,004 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:17,437 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:18,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:19,438 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:20,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:20,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:21,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:21,439 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:22,574 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:22,584 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,440 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:24,603 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:24,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:26,157 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:27,441 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:27,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,442 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:29,554 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:30,398 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:25:30,399 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:25:30,400 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:25:30,400 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:25:30,442 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:25:31,401 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:31,443 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:32,443 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:32,632 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:32,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,242 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:36,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,273 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,637 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:37,445 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:39,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:39,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:39,445 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:41,334 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:41,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:41,446 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:41,648 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:43,447 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:44,221 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:44,233 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:44,716 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:25:44,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:45,448 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:46,257 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,653 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:47,448 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:48,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:48,304 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,449 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:51,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,093 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,450 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:51,654 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:53,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:53,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:53,450 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:55,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,451 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:57,320 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:57,451 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:57,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:57,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:59,452 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:25:59,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:59,946 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:59,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,452 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:02,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:02,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:03,315 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:03,453 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:04,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:04,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:05,454 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:07,455 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:07,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:07,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:08,316 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:09,456 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:10,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:10,029 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,456 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:13,457 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:13,517 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:13,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,982 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:14,718 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:26:14,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:15,458 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:15,674 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:26:15,675 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:26:15,675 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:26:15,675 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:26:16,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,301 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,458 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:26:17,459 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:18,342 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,352 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,463 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:19,463 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:19,629 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:20,398 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:20,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:21,464 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:23,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,465 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:24,634 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:25,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,171 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,465 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:27,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,225 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,466 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:29,466 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:29,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:29,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:29,974 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:30,009 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:31,468 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:32,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:32,048 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,468 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:34,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:34,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:35,287 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:35,470 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:36,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:36,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:37,471 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:38,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:38,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:39,471 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:40,296 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:41,472 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:41,950 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:41,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,472 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:43,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,719 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:26:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:45,473 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:46,290 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:46,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:46,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:47,474 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:49,474 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:50,538 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:50,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:50,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:51,475 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:51,973 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:52,596 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:52,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,476 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:55,451 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,476 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:56,975 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:26:56,977 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:26:56,977 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:26:56,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:56,977 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:26:57,477 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:26:57,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:57,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:58,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:59,478 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:26:59,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:59,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:01,479 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:02,325 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,340 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,770 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:03,479 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:04,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:04,373 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:05,480 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:07,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,481 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:08,434 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:09,465 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,481 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:11,482 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:12,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:12,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:13,442 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:13,483 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:14,417 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,721 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:27:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:14,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:14,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:15,483 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:16,452 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:16,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:17,484 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:18,467 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:19,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,485 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:21,311 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,486 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:23,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,399 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,486 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:24,462 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:25,487 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:26,744 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:26,767 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:26,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:27,488 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:29,466 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:29,488 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:29,527 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:29,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:29,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:31,489 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:31,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,567 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:33,489 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:34,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:34,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:35,130 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:35,490 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:36,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:37,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:38,459 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:27:38,460 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:27:38,461 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:27:38,461 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:27:38,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:27:39,491 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:39,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:39,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:40,492 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:41,125 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:41,493 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:41,555 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,567 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,493 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:44,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:44,450 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:44,722 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:27:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:44,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:45,494 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:46,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,789 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:47,496 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:49,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,371 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,496 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:51,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,402 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,497 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:52,612 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:53,498 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:54,386 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,399 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:55,498 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:56,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:56,432 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,499 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:58,272 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:59,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,451 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,500 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:27:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:59,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:59,739 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:01,473 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:01,483 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:01,501 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:03,501 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:04,270 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:04,996 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:05,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:05,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:05,502 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:07,503 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:07,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,893 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,274 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:09,503 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:09,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:11,504 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:12,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:13,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:13,505 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:14,296 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:14,724 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:28:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:14,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:14,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:15,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,061 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,506 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:17,506 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:18,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:18,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:19,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:19,938 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:19,952 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:28:19,953 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:28:19,954 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:28:19,954 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:28:20,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:28:21,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:22,508 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:22,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:22,958 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:23,508 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:24,951 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:24,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,509 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:27,509 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:27,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:27,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:29,510 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:29,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:29,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:30,009 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:30,944 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:31,511 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:32,040 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:32,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,511 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:34,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:34,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:35,512 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:36,591 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:36,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:36,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:37,513 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:38,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,958 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:39,514 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:41,514 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:42,176 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:42,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:42,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:42,583 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:43,515 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:44,725 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:28:44,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:44,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:45,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,058 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,516 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:47,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:47,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:47,517 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:48,261 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:49,517 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:50,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:51,518 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:52,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:52,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:53,518 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:54,148 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:55,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,519 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:57,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:57,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:57,520 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:59,521 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:28:59,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:59,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:59,914 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:00,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:01,488 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:29:01,490 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:29:01,490 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:29:01,491 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:29:01,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:29:01,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:02,072 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:02,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:02,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:03,522 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:04,109 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:04,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:05,523 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:05,803 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:06,857 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:06,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:07,524 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:08,894 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:08,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,525 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:10,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:10,938 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:11,483 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:11,526 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:13,527 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:13,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:13,717 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:14,727 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:29:14,738 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:14,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:15,527 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:15,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:15,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,471 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:17,529 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:19,333 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:19,356 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,366 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,529 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:21,530 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:22,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:23,132 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:23,531 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:24,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:24,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:25,531 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:26,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:27,532 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:28,133 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:29,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,199 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,533 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:29,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:29,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:31,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,264 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,533 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:33,133 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:33,534 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:34,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:34,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:35,534 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:36,263 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:37,535 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:38,146 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:39,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,300 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,536 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:41,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,536 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:43,143 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:29:43,144 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:29:43,144 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:29:43,145 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:29:43,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:29:43,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:44,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:44,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:44,729 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:29:44,739 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:44,740 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:45,538 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:46,249 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:46,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:47,539 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:48,279 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:48,290 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:49,540 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:49,773 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:51,213 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,541 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:53,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,252 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,541 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:55,542 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:55,772 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:56,731 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:56,750 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:56,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,543 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:59,544 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:29:59,594 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:59,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:00,960 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:01,544 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:01,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:03,545 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:04,696 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:04,708 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:05,545 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:06,414 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:06,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:07,546 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:09,547 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:09,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:09,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,548 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:11,679 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:12,410 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:13,548 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:14,704 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:30:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:15,549 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:16,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:16,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:17,550 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:18,065 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:19,551 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:19,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:19,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:21,551 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:21,865 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:21,874 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,552 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:24,047 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:24,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:30:24,738 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:30:24,738 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:30:24,738 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:30:24,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:24,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:25,552 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:30:25,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:26,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:26,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:26,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:27,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:29,553 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:29,711 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:29,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:29,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:29,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:31,554 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:31,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:31,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,555 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:35,469 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:30:35,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,556 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:35,702 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:37,557 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:38,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,262 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:39,558 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:40,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:40,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:41,358 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:41,558 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:43,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,559 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:44,731 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:30:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:44,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:45,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,471 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,560 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:47,263 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:47,561 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:30:48,436 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:48,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:52,264 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:53,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:53,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:57,264 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:58,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:58,559 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:59,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:00,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:00,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:01,565 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:02,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:03,565 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:03,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:03,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:05,566 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:05,629 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:05,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:07,567 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:08,182 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:08,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:08,575 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:09,568 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:10,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:10,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,568 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:13,569 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:14,151 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:14,174 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:14,177 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:14,190 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:14,733 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:31:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:14,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:15,570 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:16,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:17,570 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:18,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:31:18,194 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:31:18,195 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:31:18,195 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:31:18,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:31:18,961 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,974 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:19,196 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:19,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:20,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:20,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:23,572 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:23,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:23,815 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:24,859 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:25,573 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:25,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:27,573 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:28,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:28,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:29,574 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:29,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:29,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:29,957 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:31,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,575 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:33,575 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:34,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:35,576 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:35,851 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:36,109 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:36,127 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:37,577 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:39,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,150 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,577 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:41,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:41,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:41,249 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:41,578 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:43,579 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:44,098 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,735 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:31:44,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:44,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:45,579 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:46,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:46,144 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:47,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:47,580 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:49,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,580 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:51,581 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:52,769 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:52,790 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:53,582 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:55,583 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:55,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:55,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,583 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:57,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,877 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:58,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:59,575 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:31:59,577 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:31:59,577 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:31:59,577 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:31:59,584 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:31:59,584 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:31:59,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:00,585 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:00,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:00,898 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:01,585 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:02,920 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:03,586 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:03,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:05,586 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:06,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:07,587 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:08,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:08,050 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,552 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:09,588 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:11,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,589 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:13,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,590 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:14,555 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:14,736 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:32:14,740 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:14,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:14,741 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:15,590 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:16,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:16,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:17,591 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:18,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,113 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:19,561 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:19,591 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:21,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:21,127 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:21,592 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:23,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,593 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:24,565 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:25,594 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:26,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:26,048 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,594 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:28,067 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:28,079 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,595 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:29,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:29,865 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:31,596 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:31,667 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:31,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:33,597 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:34,545 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:34,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:35,542 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:35,597 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:36,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:36,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:37,598 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:39,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,598 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:41,205 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:41,219 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:32:41,220 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:32:41,221 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:32:41,221 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:32:41,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:32:41,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:41,641 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:41,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:42,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:43,599 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:44,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,738 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:32:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:44,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:44,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:45,600 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:46,665 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:46,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:47,190 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:47,601 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:49,602 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:49,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,602 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:51,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:52,855 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:53,603 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:54,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:54,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,604 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:56,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:56,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:57,604 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:58,850 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:59,605 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:32:59,686 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:59,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:01,605 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:01,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:01,736 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:03,606 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:04,502 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:04,602 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:04,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:05,606 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:06,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:06,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:07,607 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:09,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:09,608 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:10,201 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:10,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:10,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:11,609 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:13,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,610 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:14,740 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:33:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:15,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,610 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:17,611 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:18,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:18,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:19,612 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:20,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,505 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:21,612 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:22,847 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:33:22,849 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:33:22,849 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:33:22,850 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:33:23,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:33:23,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:24,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:25,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,614 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:26,169 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:27,614 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:28,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:28,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:29,615 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:29,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:30,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:30,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:31,176 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:31,615 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:33,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,616 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:35,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:35,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:35,617 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:37,172 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:37,618 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:38,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:39,618 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:40,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:40,383 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:41,619 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:42,845 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:43,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,619 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:44,741 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:33:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:44,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:45,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:45,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:46,620 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:48,621 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:48,842 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:48,966 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:48,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:49,008 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:50,622 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:51,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:51,802 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:52,622 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:53,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:53,833 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:54,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:54,623 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:56,624 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:56,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:56,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,624 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:33:58,780 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,796 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:59,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:59,742 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:59,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:00,625 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:01,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:01,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:02,626 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:03,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:04,503 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:34:04,505 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:34:04,505 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:34:04,506 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:34:04,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:34:04,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:05,508 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:06,627 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:06,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:06,795 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,816 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:10,509 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:11,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:11,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:13,772 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:13,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:14,743 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:34:14,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:15,983 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:16,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:16,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:18,631 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:18,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:18,966 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:21,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:22,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:22,024 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:24,044 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:24,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:26,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:27,580 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:27,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:27,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:29,741 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:29,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:30,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:31,939 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:32,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:32,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:34,636 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:35,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:35,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,222 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:37,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:42,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,263 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:44,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:44,940 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:34:45,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,219 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,568 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:48,641 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:34:50,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,055 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,072 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:52,570 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:54,106 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:54,116 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:57,571 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:58,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:58,857 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:01,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:01,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:02,645 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:02,903 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:05,394 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:05,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:07,904 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:08,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:12,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:13,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:14,933 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:35:15,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:15,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:16,650 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:18,005 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,024 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,182 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:20,065 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,834 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:23,183 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:24,900 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,912 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:27,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:27,685 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,183 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:29,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:29,715 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:29,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:32,655 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:32,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:32,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:33,942 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:34,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:34,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,507 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:38,943 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:39,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:39,552 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,899 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:42,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:44,658 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:44,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:44,961 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:35:45,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:45,796 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:46,660 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:35:47,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:49,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:50,606 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:50,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:54,964 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:55,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:55,486 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,511 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:59,966 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:00,597 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,619 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,664 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:02,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:04,967 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:05,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:05,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:07,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:07,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:09,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:10,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,347 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:14,669 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:14,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:14,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:14,970 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:36:14,971 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:15,322 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:17,355 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:17,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:19,973 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:20,664 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:20,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:24,974 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:25,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:25,453 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,675 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:29,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:29,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:30,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,458 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:33,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:33,252 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:38,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:38,085 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,110 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:41,304 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:42,681 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:43,033 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:43,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:44,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:44,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:44,877 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:36:45,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:45,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:47,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:47,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:49,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:49,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:51,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:52,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:52,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,772 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:56,685 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:36:57,525 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:58,147 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:58,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:58,192 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:59,742 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:59,743 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:01,005 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:01,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:02,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:03,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:03,061 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,859 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,882 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,927 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:07,933 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,945 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,689 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:10,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,869 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,902 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:13,504 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:14,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:14,876 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:37:15,698 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,716 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,760 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:18,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:20,534 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:20,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:22,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:22,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:24,202 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:24,695 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:25,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:25,528 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:29,203 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:29,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:29,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:30,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,350 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:34,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:35,762 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:35,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:35,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,700 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:40,109 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:40,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,449 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:44,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:44,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:44,929 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:37:45,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,503 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,930 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:48,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:48,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:51,112 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:52,704 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:37:53,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:53,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:56,112 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:58,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:58,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:59,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:59,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:59,744 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:00,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:00,165 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:01,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:02,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:02,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:04,708 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:05,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:05,027 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:07,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:07,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:07,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:10,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:10,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:12,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:13,493 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:13,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:13,521 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:14,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:14,955 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:38:16,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:16,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:18,663 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:18,712 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:21,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,303 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,314 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:23,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:26,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:26,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,664 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:29,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:29,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:31,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:31,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:32,717 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:33,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:33,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:34,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:36,122 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:36,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:39,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:40,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:40,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:44,419 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:44,743 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:44,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:44,887 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:38:45,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:45,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:46,722 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:38:47,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:47,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:50,387 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:51,323 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:51,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:51,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:54,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:54,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,389 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:56,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:56,071 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:59,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:59,745 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:00,726 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:00,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,924 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,937 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:03,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:03,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,760 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,938 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:08,700 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:08,714 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,939 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:13,590 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:13,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:14,730 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:14,744 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:14,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:14,975 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:39:15,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,976 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:18,732 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:18,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:23,577 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:23,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:28,736 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:28,947 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:28,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:28,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:29,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:31,701 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:31,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:31,960 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:33,731 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,725 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,961 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:38,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:38,771 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:41,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:41,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:42,501 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:42,742 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:43,788 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:43,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:44,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:44,893 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:39:46,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:46,599 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:47,896 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:48,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:48,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:51,414 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:51,424 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:52,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:53,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:53,458 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,557 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,580 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,747 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:39:58,548 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:58,602 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:58,629 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:59,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:01,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:01,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,452 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:06,876 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:06,910 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:09,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:09,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:10,753 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:11,563 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:11,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:14,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:14,918 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:40:16,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,740 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:19,921 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:21,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:21,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:25,029 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:26,591 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,757 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:28,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:28,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:29,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:29,746 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:30,906 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:31,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:31,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:35,907 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:36,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:38,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:38,355 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:40,761 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:41,070 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:41,193 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:41,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:44,885 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:44,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,915 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:40:44,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:47,910 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:47,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,917 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:52,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:52,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,766 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:40:54,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:57,104 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:57,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:57,790 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:59,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:59,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,636 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,977 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:04,658 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:04,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,770 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:07,665 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:07,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:08,019 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:09,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:09,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,690 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,710 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:13,020 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:14,729 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,745 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:14,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:14,749 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,976 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:41:17,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:17,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:18,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:19,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:19,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:20,775 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:23,041 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:23,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:23,075 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,036 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:25,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:25,757 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,776 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:29,038 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:29,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:30,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:30,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,780 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:34,987 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:35,619 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:35,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,654 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,664 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,988 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:40,613 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:40,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:44,924 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:41:45,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:45,514 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:45,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:47,532 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:47,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:48,785 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:41:50,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:50,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:51,778 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:52,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:56,779 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:57,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,575 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:59,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:00,892 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:00,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:00,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,617 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:02,789 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:03,627 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:03,637 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,657 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,669 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:08,582 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:08,592 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,610 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,619 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:13,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:13,457 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:14,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:14,973 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:42:15,479 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:15,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:16,793 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:18,087 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:18,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:18,324 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,343 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,352 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,089 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:23,264 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,298 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,090 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:28,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,183 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:29,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:29,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:30,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,797 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:33,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,948 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:35,216 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:35,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,649 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:38,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,677 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:41,395 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:41,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,437 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:44,167 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:44,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:44,747 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:44,801 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:44,871 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:42:46,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:46,407 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,427 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:49,873 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:51,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:51,376 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,396 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:54,874 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:56,257 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:56,269 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,287 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,806 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:42:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:59,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:59,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:01,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:01,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:04,925 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:06,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:06,140 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:09,926 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:10,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:11,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:12,810 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:13,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:13,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:14,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:14,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:14,883 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:43:15,885 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:16,504 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:16,526 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:16,542 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:20,886 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:21,276 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:21,286 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:24,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:24,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,387 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:26,816 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:29,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:29,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:31,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:31,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:31,869 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:33,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:33,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:36,870 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:38,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:38,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:40,821 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:40,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:40,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:42,226 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:43,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:43,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:44,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:44,947 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:43:45,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,867 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:47,948 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:48,815 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:52,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:54,399 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:54,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:54,435 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:54,825 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:43:57,243 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:57,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:58,138 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:59,281 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:59,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:59,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:59,748 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:02,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:02,183 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:03,922 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:04,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:04,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:07,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:07,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:08,830 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:08,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:09,382 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:09,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,083 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:13,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:14,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:14,998 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:14,999 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:15,191 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:44:17,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:17,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:19,238 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,112 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:22,834 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:24,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:24,955 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:25,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:27,523 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:27,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:29,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:30,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,056 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:34,077 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:35,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:36,839 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:37,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:37,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:39,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:39,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:42,089 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:42,099 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:44,911 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:44:46,913 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:46,933 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:46,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:50,844 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:44:52,025 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,035 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,786 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:54,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:54,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,786 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:59,050 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,063 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:01,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:01,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:02,798 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:03,803 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:03,818 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,848 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:05,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:05,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:07,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:08,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:08,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,212 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:12,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:12,806 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:14,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:14,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:14,891 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:45:15,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:15,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:17,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:17,430 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:18,143 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:18,853 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:20,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:22,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:22,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:23,144 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:25,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,171 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:27,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:27,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:28,145 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:29,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:29,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,858 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:33,388 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:35,104 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:35,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:38,389 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:40,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:43,390 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:44,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:44,946 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:45:45,079 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:45,098 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:46,862 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:47,118 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:47,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:48,595 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:50,437 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:50,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:50,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:53,596 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:55,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:55,162 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,145 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:58,798 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:58,866 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:45:59,747 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:00,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:00,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:03,152 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:03,163 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:04,380 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:04,868 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:05,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:05,195 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:09,380 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:09,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:10,001 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:10,870 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:12,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:12,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:14,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:14,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,863 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:14,978 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:46:17,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:17,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,761 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,979 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:22,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:22,723 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,874 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:25,316 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:28,218 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:28,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:28,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:29,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:30,902 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:30,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:30,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:32,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,769 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:35,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:37,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,647 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:40,879 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:41,547 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:42,686 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,698 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:44,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:44,979 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:46:45,585 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:45,606 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:46,980 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:47,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:47,652 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:50,443 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:50,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:51,981 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:52,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,508 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,883 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:46:55,682 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,469 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:57,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:59,749 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:59,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,824 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,897 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:02,930 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,590 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:06,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:06,886 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:08,485 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:09,390 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,407 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:11,426 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:11,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:13,486 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:14,365 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:14,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:14,884 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:47:16,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:16,440 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,472 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,490 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,885 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:20,890 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:21,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:21,725 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,744 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:24,370 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:25,778 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:25,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:29,371 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:29,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:29,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:30,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:30,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,866 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:34,893 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:35,274 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:35,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:35,694 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,275 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:40,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,519 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:44,545 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,559 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:44,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:44,898 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:47:45,899 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:46,578 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:46,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:48,898 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:47:49,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:49,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:51,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:51,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:51,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:56,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:57,015 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,748 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:59,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:59,805 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,866 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,897 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:02,902 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:04,672 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:04,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:06,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:06,761 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:07,185 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:08,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:08,806 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,924 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:12,186 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:13,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:13,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:14,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:14,931 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:48:15,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:16,017 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:16,906 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:17,985 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:18,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:18,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,565 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:22,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:25,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:25,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,986 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:29,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:30,910 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:31,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,111 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,927 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:35,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:35,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,928 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:40,308 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:40,331 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:43,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:43,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:44,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:44,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:44,896 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:48:44,914 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:45,387 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:45,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,340 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:49,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:50,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:50,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:54,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:55,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,282 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,319 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:58,919 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:48:59,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:59,750 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:00,878 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:01,072 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:01,121 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:01,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:05,879 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:06,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:06,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,133 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:10,880 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:11,201 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:11,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:12,923 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:14,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:14,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:14,968 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:49:15,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:16,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:16,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:20,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:21,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:21,168 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:25,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:26,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,927 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:28,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:28,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:29,749 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:29,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:31,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,969 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:33,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,579 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,342 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,362 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:40,098 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:40,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:40,136 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:40,931 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:42,080 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:43,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:43,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:44,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:44,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:49:45,206 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,225 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,266 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:47,953 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:50,380 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,393 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,425 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:54,445 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:54,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:54,935 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:49:57,350 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:58,746 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:59,394 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:59,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:59,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:02,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:02,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:03,965 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:04,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,724 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,783 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,853 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:09,939 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:11,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:14,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:14,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:14,887 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:50:14,889 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:17,741 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:18,561 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:18,818 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:19,888 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:20,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,845 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,942 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:24,892 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:25,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:25,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:28,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:28,717 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:29,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:29,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:29,935 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:30,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:30,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,784 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,936 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:35,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:35,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:37,946 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:38,004 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:38,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,045 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:40,078 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:42,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:42,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:44,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:44,752 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:44,916 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:50:44,931 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:44,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:45,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:48,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:48,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:50,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:50,083 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:51,447 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:51,949 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:50:52,953 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:52,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,449 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:56,490 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:56,510 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,400 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:59,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:59,751 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:01,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,985 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:04,376 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:04,386 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:05,954 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:06,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:06,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:07,384 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:09,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:09,431 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:12,385 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:14,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:14,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:14,752 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:14,933 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:51:16,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,457 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:17,817 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:19,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,958 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:21,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:21,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:22,858 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:24,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,293 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,313 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:27,858 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:29,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,750 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:31,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:31,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:33,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:33,962 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:34,871 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:34,899 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:37,718 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:37,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:38,736 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:39,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:42,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:42,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:43,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:44,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:44,939 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:51:47,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:47,609 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:47,966 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:51:49,417 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:49,630 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,573 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,418 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:54,607 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,482 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,420 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:59,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:59,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:01,971 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:02,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,530 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,540 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,899 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:07,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,493 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:12,890 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:12,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:12,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:13,974 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:14,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:14,947 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:52:14,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:15,731 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:15,745 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,773 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,949 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:20,612 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:20,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,645 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:25,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:25,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,661 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:29,970 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:29,979 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:30,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:30,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,611 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,650 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,971 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:37,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:39,972 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:42,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,401 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:43,983 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:44,446 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:44,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:44,899 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:52:45,900 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:47,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,901 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:50,967 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:50,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:53,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:53,871 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,895 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,987 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:52:56,529 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:58,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:58,884 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:59,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:59,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:00,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:01,915 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:03,864 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:03,877 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:06,916 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:08,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:08,920 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:09,990 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:10,941 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:10,956 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:12,718 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:13,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:14,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:14,949 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:53:15,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:17,950 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:18,957 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:18,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,834 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:23,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,989 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,994 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:26,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:28,835 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:29,387 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:29,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:29,425 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:29,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:32,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:32,201 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:33,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:34,224 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:34,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:37,998 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:39,121 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,628 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:42,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:42,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,075 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,629 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:44,751 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:44,753 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:44,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:53:46,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:46,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,952 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,954 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:51,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:51,935 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:52,002 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:53:53,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,981 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:55,527 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:56,821 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:56,844 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,891 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:00,918 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:01,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:01,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,897 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:06,005 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:06,501 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:08,454 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:08,471 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:08,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:10,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:10,515 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,502 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:13,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:13,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:14,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:14,896 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:54:15,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:15,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,898 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:18,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:20,009 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:20,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:20,334 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:22,513 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:23,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:25,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:25,160 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:27,513 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:27,966 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:27,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:29,998 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:30,013 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,997 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:33,137 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:34,013 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:35,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:35,027 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:37,839 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:37,849 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:38,138 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:39,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:39,882 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:42,825 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:42,837 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:43,139 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:44,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:44,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:44,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:44,993 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:54:46,015 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:54:46,393 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:46,411 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:48,442 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:48,995 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:49,466 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,487 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:53,996 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:54,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,361 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,377 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,190 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:59,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:59,756 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:00,020 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:01,240 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:01,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,893 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:06,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:06,197 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,038 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,894 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:11,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:11,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,023 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:14,056 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,752 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:14,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:14,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:14,944 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:55:14,944 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:16,088 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:16,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,076 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,945 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:21,104 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:21,115 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,460 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:24,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,501 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,946 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:27,303 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:28,027 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:29,346 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,359 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:29,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:30,909 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:32,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:32,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:35,910 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:36,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:36,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,012 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,839 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:41,890 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:42,031 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:43,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:43,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:44,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:44,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:44,755 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:44,980 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:55:46,707 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:46,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:46,981 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:48,764 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:48,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,595 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,622 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,982 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:53,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:53,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,035 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:55:56,623 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:57,446 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:58,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:59,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:59,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:59,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:59,970 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:02,946 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:03,801 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:03,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:03,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:08,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:10,040 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:10,301 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:10,546 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,284 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:13,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,569 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:14,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:14,908 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:14,909 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:15,129 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:56:15,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,722 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:19,131 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:21,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:21,853 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:23,875 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:23,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:24,045 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:24,266 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:26,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:26,935 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,964 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:29,267 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:29,753 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:29,754 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:29,754 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:30,987 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:31,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,675 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,695 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:34,978 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:35,721 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:35,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,048 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:38,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,676 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,450 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:40,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,716 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,081 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:44,107 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,120 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,764 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:44,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:44,775 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:44,905 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:56:44,940 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: metric +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:56:44,941 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:56:44,943 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:56:45,050 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:56:45,947 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:46,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:46,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:46,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:47,051 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/config.yaml +2024-03-27 19:56:48,906 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:51,123 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:52,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:52,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:54,053 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:56:55,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:56,941 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:57,297 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:58,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:59,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:59,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:59,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:02,034 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,170 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,931 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:04,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:04,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:06,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:07,131 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:07,141 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,056 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:08,435 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:09,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:09,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:10,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:12,058 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:12,202 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:12,211 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,441 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:14,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:14,231 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:14,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:14,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:14,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:14,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:14,952 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:57:16,059 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:17,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:18,060 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:19,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:19,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:19,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:20,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:21,382 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:21,394 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:22,061 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:24,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:25,084 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:26,062 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:26,780 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:26,797 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:26,806 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:28,064 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:28,829 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:28,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:29,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:29,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:29,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:30,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:31,075 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:31,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:31,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,065 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:33,653 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:33,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:34,066 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:36,067 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:36,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:36,681 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:36,725 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:38,068 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:38,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:38,724 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:40,754 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:42,069 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:42,689 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:43,518 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,528 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:44,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:44,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:44,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:44,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:44,984 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:57:45,023 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:57:45,024 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:57:45,024 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:57:45,024 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:57:45,070 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:57:45,547 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:45,556 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:46,071 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:48,072 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:48,340 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:48,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,589 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,073 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:50,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,638 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:52,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,693 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:54,074 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:54,333 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:55,385 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:55,405 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:56,075 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:57,426 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:57,441 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:58,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:57:59,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:59,483 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:59,760 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:59,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:59,763 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:59,765 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:00,076 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:02,077 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:02,758 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:02,779 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,792 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:04,991 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:05,610 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:05,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:06,078 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:07,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:07,654 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:08,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:10,079 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:10,687 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,968 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:12,080 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:12,733 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:12,747 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,081 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:14,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:14,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:14,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:14,900 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 19:58:15,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:15,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:16,082 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:16,613 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:17,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:17,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:18,083 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:20,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:20,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:20,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:22,084 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:22,597 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:22,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:22,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:24,085 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:26,066 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:26,090 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,615 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:58:26,616 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 19:58:26,616 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:58:26,617 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:58:27,086 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 19:58:27,618 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:28,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:28,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:28,139 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:29,763 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:29,764 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:29,805 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:30,087 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 19:58:30,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,170 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported