diff --git "a/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" "b/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" --- "a/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" +++ "b/wandb/run-20240327_190513-7p2x8a0l/logs/debug-internal.log" @@ -25172,3 +25172,1801 @@ 2024-03-27 23:25:14,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported 2024-03-27 23:25:14,202 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report 2024-03-27 23:25:15,284 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:25:15,512 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:25:16,227 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:16,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:17,285 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:25:18,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:25:18,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:25:18,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:25:19,019 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:19,040 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:19,286 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:25:20,201 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:21,083 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:21,093 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:24,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:24,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:25,201 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:27,681 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:25:27,700 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:27,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:29,731 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:29,741 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:30,356 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:31,289 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:25:32,459 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:32,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:33,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:25:33,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:25:33,052 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:25:34,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:34,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:36,258 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:37,305 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:37,320 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:39,355 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:39,364 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:41,259 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:42,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:42,170 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:44,188 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:44,201 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:45,294 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:25:45,514 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:25:46,515 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:47,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:47,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:48,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:25:48,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:25:48,052 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:25:49,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:49,247 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:51,991 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:52,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:52,182 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:54,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:54,054 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:56,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:56,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:58,050 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:25:58,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:58,945 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:25:59,298 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:26:01,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:01,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:03,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:26:03,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:26:03,052 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:26:03,250 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:05,351 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:26:05,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:05,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:08,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:08,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:08,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:10,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:10,351 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:13,302 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:26:13,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:13,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:14,199 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:15,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:15,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:15,515 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:26:18,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:26:18,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:26:18,052 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:26:18,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:18,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:19,282 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:20,287 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:20,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:23,340 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:23,356 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:24,282 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:25,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:25,388 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:27,305 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:26:28,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:28,383 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:30,130 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:30,408 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:30,419 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:33,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:26:33,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:26:33,052 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:26:33,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:33,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:35,234 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:35,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:35,269 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:38,123 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:38,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:40,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:40,178 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:40,412 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:41,310 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:26:43,719 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:26:43,739 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:43,750 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:45,413 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:45,518 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:26:46,489 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:46,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:48,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:26:48,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:26:48,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:26:48,528 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:48,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:51,212 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:51,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:51,342 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:53,361 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:53,371 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:55,315 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:26:56,330 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:56,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:57,094 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:26:58,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:26:58,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:01,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:01,192 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:02,096 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:03,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:27:03,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:27:03,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:27:03,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:03,220 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:06,039 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:06,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:07,196 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:08,067 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:08,078 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:09,320 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:27:11,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:11,042 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:12,237 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:13,065 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:13,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:15,519 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:27:15,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:15,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:17,520 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:17,923 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:17,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:18,051 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:27:18,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:27:18,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:27:21,281 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:27:21,301 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:21,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:23,131 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:23,324 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:27:24,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:24,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:26,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:26,194 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:28,132 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:28,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:28,994 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:31,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:31,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:33,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:27:33,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:27:33,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:27:33,212 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:33,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:34,001 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:35,327 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:27:36,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:36,033 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:38,861 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:38,862 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:38,871 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:40,889 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:40,903 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:43,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:43,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:43,862 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:45,520 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:27:45,871 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:45,882 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:48,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:27:48,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:27:48,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:27:48,700 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:48,709 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:49,280 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:49,331 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:27:50,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:50,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:53,524 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:53,536 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:54,282 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:27:55,558 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:55,570 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:58,977 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:27:58,999 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:59,010 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:27:59,283 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:01,788 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:01,800 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:03,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:28:03,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:28:03,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:28:03,336 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:28:03,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:03,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:05,247 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:06,604 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:06,614 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:08,632 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:08,642 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:10,248 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:11,602 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:11,612 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:13,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:13,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:15,249 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:15,522 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:28:16,618 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:16,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:17,340 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:28:18,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:28:18,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:28:18,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:28:18,660 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:18,670 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:21,094 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:21,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:21,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:23,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:23,521 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:26,095 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:26,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:26,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:28,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:28,350 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:31,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:31,312 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:31,344 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:28:31,390 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:33,052 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:28:33,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:28:33,053 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:28:33,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:33,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:36,600 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:28:36,617 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:36,628 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:37,220 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:39,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:39,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:41,404 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:41,416 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:42,221 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:44,402 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:44,413 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:45,348 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:28:45,524 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:28:46,435 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:46,447 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:47,525 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:48,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:28:48,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:28:48,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:28:49,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:49,263 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:51,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:51,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:53,185 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:54,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:54,247 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:56,269 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:56,282 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:58,732 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:28:59,250 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:59,260 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:28:59,352 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:01,281 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:01,291 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:03,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:29:03,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:29:03,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:29:04,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:04,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:04,258 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:06,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:06,140 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:09,027 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:09,041 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:09,355 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:10,102 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:11,061 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:11,074 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:13,357 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:14,360 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:29:14,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:14,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:15,526 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:29:15,528 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:17,369 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:17,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:18,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:29:18,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:29:18,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:29:19,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:19,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:21,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:22,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:22,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:23,360 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:24,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:24,323 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:26,462 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:27,227 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:27,237 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:29,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:29,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:31,463 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:32,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:32,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:33,053 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:29:33,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:29:33,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:29:34,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:34,114 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:36,510 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:37,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:37,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:37,365 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:39,121 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:39,130 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:41,518 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:41,930 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:41,944 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:43,962 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:43,971 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:45,527 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:29:46,528 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:46,784 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:46,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:48,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:29:48,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:29:48,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:29:48,818 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:48,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:51,368 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:29:52,156 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:29:52,175 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:52,193 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:52,498 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:54,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:54,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:56,946 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:56,957 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:57,498 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:29:59,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:29:59,769 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:01,788 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:01,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:02,499 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:03,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:30:03,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:30:03,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:30:04,756 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:04,768 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:05,372 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:30:06,790 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:06,802 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:08,290 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:09,593 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:09,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:11,624 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:11,635 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:13,291 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:14,535 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:14,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:15,529 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:30:16,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:16,583 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:18,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:30:18,054 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:30:18,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:30:19,237 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:19,376 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:30:19,433 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:19,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:21,467 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:21,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:24,238 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:24,262 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:24,274 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:26,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:26,307 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:29,239 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:29,691 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:30:29,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:29,726 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:32,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:32,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:33,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:30:33,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:30:33,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:30:33,380 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:30:34,644 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:34,653 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:35,234 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:37,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:37,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:39,509 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:39,520 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:40,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:42,482 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:42,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:44,512 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:44,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:45,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:45,530 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:30:47,385 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:30:47,469 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:47,479 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:48,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:30:48,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:30:48,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:30:49,500 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:49,514 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:50,766 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:52,295 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:52,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:54,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:54,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:55,767 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:30:57,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:57,287 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:59,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:30:59,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:00,931 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:01,389 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:31:02,268 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:02,285 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:03,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:31:03,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:31:03,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:31:04,309 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:04,319 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:06,188 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:07,582 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:31:07,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:07,614 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:10,349 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:10,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:11,189 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:12,380 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:12,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:15,197 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:15,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:15,393 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:31:15,532 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:31:16,533 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:17,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:17,239 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:18,054 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:31:18,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:31:18,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:31:20,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:20,032 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:22,051 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:22,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:22,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:25,089 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:25,099 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:27,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:27,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:27,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:29,398 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:31:29,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:29,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:32,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:32,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:32,907 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:33,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:31:33,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:31:33,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:31:34,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:34,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:36,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:36,960 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:38,230 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:39,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:39,774 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:41,794 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:41,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:43,401 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:31:43,924 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:45,216 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:31:45,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:45,243 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:45,534 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:31:47,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:48,002 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:48,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:31:48,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:31:48,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:31:49,218 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:50,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:50,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:53,062 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:53,072 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:54,219 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:31:55,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:55,104 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:57,406 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:31:58,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:58,079 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:31:59,946 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:00,101 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:00,117 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:02,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:02,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:03,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:32:03,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:32:03,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:32:04,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:04,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:05,245 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:07,924 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:07,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:09,958 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:09,967 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:10,735 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:11,410 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:32:12,905 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:12,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:14,940 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:14,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:15,536 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:32:16,537 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:17,843 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:17,857 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:18,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:32:18,055 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:32:18,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:32:19,881 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:19,891 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:22,200 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:23,326 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:32:23,344 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:23,356 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:25,415 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:32:26,098 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:26,110 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:27,470 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:28,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:28,144 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:31,053 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:31,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:32,471 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:33,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:32:33,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:32:33,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:32:33,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:33,093 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:35,907 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:35,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:37,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:37,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:38,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:38,419 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:32:40,770 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:40,786 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:42,808 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:42,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:43,242 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:45,537 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:32:45,872 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:45,889 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:47,908 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:47,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:48,055 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:32:48,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:32:48,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:32:49,937 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:49,951 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:50,329 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:52,423 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:32:52,804 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:52,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:54,842 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:54,855 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:55,889 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:32:57,785 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:32:57,798 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:00,890 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:01,376 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:33:01,397 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:01,407 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:03,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:33:03,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:33:03,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:33:04,451 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:04,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:06,222 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:06,427 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:33:06,494 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:06,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:09,381 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:09,392 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:11,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:11,412 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:11,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:13,444 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:13,462 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:15,539 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:33:16,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:16,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:16,540 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:18,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:33:18,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:33:18,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:33:18,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:18,223 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:20,431 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:33:20,984 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:21,013 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:22,238 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:23,036 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:23,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:25,868 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:25,884 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:27,239 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:27,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:27,939 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:30,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:30,715 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:32,738 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:32,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:32,854 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:33,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:33:33,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:33:33,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:33:34,435 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:33:35,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:35,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:38,244 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:39,297 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:33:39,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:39,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:42,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:42,261 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:43,245 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:44,280 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:44,291 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:45,541 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:33:47,127 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:47,137 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:48,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:33:48,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:33:48,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:33:48,439 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:33:49,156 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:49,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:49,201 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:51,973 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:51,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:54,004 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:54,013 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:54,202 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:33:56,807 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:56,819 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:58,836 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:58,851 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:33:59,204 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:00,444 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:34:01,934 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:01,947 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:03,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:34:03,056 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:34:03,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:34:03,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:03,980 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:04,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:07,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:07,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:09,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:09,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:09,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:12,084 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:12,095 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:14,114 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:14,132 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:14,506 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:15,542 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:34:16,449 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:34:17,403 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:34:17,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:17,433 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:18,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:34:18,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:34:18,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:34:20,163 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:20,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:20,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:22,197 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:22,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:25,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:25,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:25,237 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:27,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:27,248 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:28,453 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:34:30,147 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:30,165 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:31,164 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:32,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:32,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:33,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:34:33,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:34:33,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:34:35,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:35,235 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:36,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:37,277 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:37,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:40,207 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:40,227 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:41,242 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:42,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:42,267 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:44,458 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:34:45,306 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:45,327 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:45,544 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:34:46,545 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:47,365 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:47,375 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:48,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:34:48,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:34:48,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:34:50,132 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:50,158 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:52,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:52,190 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:52,268 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:55,476 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:34:55,495 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:55,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:56,461 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:34:57,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:34:58,216 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:34:58,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:00,249 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:00,259 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:02,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:02,302 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:02,422 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:03,056 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:35:03,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:35:03,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:35:04,976 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:04,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:07,007 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:07,016 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:08,257 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:09,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:09,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:10,466 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:35:11,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:11,887 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:13,393 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:14,696 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:14,708 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:15,545 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:35:16,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:16,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:18,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:35:18,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:35:18,097 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:35:19,213 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:19,659 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:19,672 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:21,693 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:21,703 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:24,471 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:35:24,532 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:24,542 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:25,191 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:26,562 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:26,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:29,562 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:29,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:30,192 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:33,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:35:33,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:35:33,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:35:33,064 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:35:33,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:33,090 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:35,272 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:36,166 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:36,177 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:38,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:38,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:38,476 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:35:41,050 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:41,060 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:41,077 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:43,114 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:43,124 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:45,546 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:35:46,024 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:46,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:46,548 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:48,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:35:48,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:35:48,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:35:48,087 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:48,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:50,872 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:50,893 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:51,844 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:52,481 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:35:52,925 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:52,934 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:55,679 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:55,698 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:56,845 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:35:57,736 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:35:57,746 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:00,511 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:00,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:01,845 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:02,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:02,574 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:03,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:36:03,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:36:03,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:36:05,387 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:05,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:06,485 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:36:07,135 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:07,432 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:07,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:10,859 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:36:10,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:10,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:12,136 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:13,606 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:13,621 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:15,548 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:36:15,641 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:15,651 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:17,550 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:18,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:36:18,057 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:36:18,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:36:18,475 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:18,492 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:20,490 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:36:20,539 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:20,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:22,930 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:23,436 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:23,453 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:25,488 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:25,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:27,930 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:28,294 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:28,310 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:30,349 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:30,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:32,931 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:33,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:36:33,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:36:33,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:36:33,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:33,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:34,494 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:36:35,246 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:35,257 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:38,163 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:38,188 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:38,263 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:40,212 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:40,229 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:42,985 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:43,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:43,263 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:45,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:45,044 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:45,549 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:36:48,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:36:48,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:36:48,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:36:48,321 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:36:48,345 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:48,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:48,498 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:36:49,246 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:51,198 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:51,208 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:53,226 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:53,236 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:54,247 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:36:56,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:56,062 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:58,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:58,105 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:36:59,247 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:00,909 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:00,928 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:02,503 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:37:02,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:02,979 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:03,057 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:37:03,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:37:03,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:37:05,192 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:05,713 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:05,737 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:07,763 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:07,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:10,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:10,544 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:10,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:12,587 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:12,603 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:15,366 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:15,551 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:37:15,554 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:15,578 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:16,507 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:37:17,616 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:17,625 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:18,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:37:18,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:37:18,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:37:20,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:20,391 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:21,223 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:22,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:22,423 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:25,678 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:37:25,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:25,706 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:26,224 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:28,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:28,464 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:30,482 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:30,491 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:30,511 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:37:31,606 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:33,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:37:33,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:37:33,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:37:33,316 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:33,335 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:35,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:35,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:37,273 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:38,114 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:38,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:40,176 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:40,186 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:42,274 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:43,134 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:43,161 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:44,516 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:37:45,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:45,202 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:45,553 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:37:47,554 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:48,000 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:48,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:48,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:37:48,058 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:37:48,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:37:50,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:50,080 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:52,962 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:52,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:53,007 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:55,031 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:55,049 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:57,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:57,883 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:58,506 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:37:58,520 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:37:59,904 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:37:59,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:03,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:38:03,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:38:03,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:38:03,154 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:38:03,174 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:03,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:04,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:06,043 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:06,052 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:08,071 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:08,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:09,242 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:10,968 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:10,986 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:12,525 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:38:13,006 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:13,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:14,495 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:15,554 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:38:15,923 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:15,939 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:17,962 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:17,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:18,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:38:18,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:38:18,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:38:20,505 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:20,751 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:20,771 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:22,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:22,805 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:25,742 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:25,762 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:26,332 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:26,529 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:38:27,806 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:27,817 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:30,701 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:30,719 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:31,333 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:32,738 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:32,753 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:33,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:38:33,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:38:33,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:38:35,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:35,564 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:37,185 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:37,605 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:37,614 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:40,533 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:38:40,992 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:38:41,011 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:41,023 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:42,870 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:43,879 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:43,890 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:45,556 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:38:45,915 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:45,929 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:48,022 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:48,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:38:48,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:38:48,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:38:48,765 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:48,794 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:50,823 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:50,844 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:52,537 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:38:53,692 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:38:53,766 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:53,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:55,828 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:55,838 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:58,579 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:58,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:38:58,694 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:00,626 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:00,645 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:03,058 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:39:03,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:39:03,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:39:03,439 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:03,456 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:04,286 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:05,497 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:05,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:06,542 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:39:08,420 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:08,438 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:09,352 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:10,465 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:10,481 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:13,249 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:13,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:14,353 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:15,311 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:15,321 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:15,557 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:39:18,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:39:18,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:39:18,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:39:18,621 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:39:18,646 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:18,656 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:20,177 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:21,468 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:21,478 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:22,547 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:39:23,504 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:23,516 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:25,178 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:26,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:26,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:28,396 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:28,409 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:30,179 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:31,184 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:31,205 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:33,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:39:33,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:39:33,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:39:33,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:33,241 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:34,551 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:39:35,192 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:36,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:36,210 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:38,244 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:38,254 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:40,192 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:41,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:41,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:43,086 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:43,101 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:45,193 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:45,559 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:39:45,880 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:45,903 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:47,923 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:47,943 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:48,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:39:48,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:39:48,101 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:39:48,557 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:39:50,237 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:50,854 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:50,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:52,894 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:52,917 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:55,238 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:39:56,153 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:39:56,172 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:56,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:58,936 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:39:58,948 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:00,239 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:00,969 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:00,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:02,561 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:40:03,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:40:03,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:40:03,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:40:03,962 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:03,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:06,021 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:06,030 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:06,197 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:08,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:08,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:10,911 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:10,921 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:11,198 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:13,850 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:13,873 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:15,562 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:40:15,918 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:15,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:16,563 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:16,566 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:40:18,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:40:18,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:40:18,059 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:40:18,769 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:18,789 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:20,824 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:20,835 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:22,256 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:23,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:23,639 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:25,679 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:25,688 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:27,257 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:28,455 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:28,474 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:30,505 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:30,525 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:30,571 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:40:32,992 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:33,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:40:33,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:40:33,060 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:40:33,929 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:40:33,955 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:33,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:36,802 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:36,812 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:38,260 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:38,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:38,841 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:41,678 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:41,699 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:42,575 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:40:43,720 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:43,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:43,803 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:45,563 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:40:46,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:46,537 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:48,059 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:40:48,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:40:48,060 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:40:48,559 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:48,576 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:49,217 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:51,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:51,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:53,365 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:53,380 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:54,218 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:40:56,476 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:56,501 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:56,586 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:40:58,760 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:58,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:40:59,629 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:01,022 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:01,048 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:03,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:41:03,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:41:03,060 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:41:03,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:03,802 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:05,222 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:05,827 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:05,846 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:08,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:08,662 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:10,590 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:41:10,823 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:12,129 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:41:12,146 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:12,157 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:15,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:15,255 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:15,564 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:41:16,566 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:17,275 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:17,287 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:18,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:41:18,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:41:18,060 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:41:20,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:20,183 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:22,204 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:22,214 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:22,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:24,595 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:41:25,153 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:25,173 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:27,196 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:27,215 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:27,697 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:30,003 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:30,018 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:32,057 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:32,067 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:32,698 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:33,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:41:33,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:41:33,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:41:34,840 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:34,865 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:36,886 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:36,901 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:38,462 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:38,600 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:41:39,926 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:39,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:41,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:41,988 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:43,463 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:44,896 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:44,922 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:45,566 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:41:46,965 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:46,977 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:48,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:41:48,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:41:48,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:41:49,208 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:50,302 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:41:50,324 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:50,336 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:52,604 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:41:53,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:53,102 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:54,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:55,128 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:55,137 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:57,951 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:57,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:41:59,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:41:59,992 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:00,005 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:02,758 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:02,782 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:03,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:42:03,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:42:03,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:42:04,801 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:04,813 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:05,155 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:06,609 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:42:07,620 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:07,640 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:09,684 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:09,697 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:10,156 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:12,460 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:12,480 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:14,503 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:14,522 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:15,156 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:15,567 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:42:17,298 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:17,318 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:18,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:42:18,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:42:18,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:42:19,357 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:19,367 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:20,241 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:20,613 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:42:22,230 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:22,253 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:24,279 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:24,296 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:25,242 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:27,517 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:42:27,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:27,548 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:30,244 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:30,304 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:30,315 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:32,339 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:32,353 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:32,617 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:42:33,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:42:33,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:42:33,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:42:35,169 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:35,189 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:35,256 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:37,208 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:37,220 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:40,028 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:40,047 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:40,256 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:42,082 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:42,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:44,876 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:44,900 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:45,257 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:45,569 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:42:46,621 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:42:46,919 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:46,932 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:48,060 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:42:48,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:42:48,061 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:42:49,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:49,847 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:50,290 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:51,881 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:51,895 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:54,680 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:54,694 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:55,291 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:42:56,733 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:56,745 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:59,549 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:42:59,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:00,625 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:43:01,082 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:01,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:01,610 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:03,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:43:03,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:43:03,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:43:04,819 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:43:04,839 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:04,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:06,281 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:07,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:07,572 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:09,590 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:09,601 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:11,282 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:12,598 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:12,619 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:14,629 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:43:14,645 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:14,662 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:15,570 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:43:16,572 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:17,432 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:17,448 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:18,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:43:18,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:43:18,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:43:19,490 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:19,502 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:22,250 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:22,270 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:22,272 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:24,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:24,308 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:27,129 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:27,148 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:27,881 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:28,633 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:43:29,197 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:29,208 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:32,046 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:32,068 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:32,883 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:33,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:43:33,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:43:33,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:43:34,092 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:34,109 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:36,861 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:36,881 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:38,207 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:38,902 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:38,914 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:42,187 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:43:42,206 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:42,218 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:42,637 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:43:43,567 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:44,949 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:44,959 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:45,572 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:43:46,978 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:46,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:48,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:43:48,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:43:48,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:43:49,200 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:49,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:49,810 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:51,830 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:51,848 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:54,200 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:54,648 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:54,667 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:56,642 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:43:56,704 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:56,713 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:59,292 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:43:59,477 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:43:59,496 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:01,518 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:01,538 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:03,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:44:03,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:44:03,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:44:04,292 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:04,313 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:05,266 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:06,354 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:06,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:09,182 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:09,203 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:10,511 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:10,646 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:44:11,221 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:11,234 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:14,270 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:14,288 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:15,511 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:15,573 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:44:16,332 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:16,345 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:18,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:44:18,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:44:18,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:44:19,682 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:44:19,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:19,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:21,227 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:22,588 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:22,600 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:22,650 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:44:24,618 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:24,633 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:26,655 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:26,666 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:27,021 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:29,473 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:29,484 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:31,506 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:31,517 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:32,022 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:33,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:44:33,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:44:33,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:44:34,329 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:34,341 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:36,363 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:36,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:36,654 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:44:37,340 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:38,410 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:38,429 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:41,325 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:41,337 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:42,341 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:43,358 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:43,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:45,575 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:44:46,348 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:46,361 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:47,577 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:48,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:44:48,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:44:48,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:44:48,379 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:48,389 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:50,658 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:44:51,216 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:51,228 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:53,245 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:53,255 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:53,350 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:56,623 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:44:56,643 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:56,653 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:58,350 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:44:59,403 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:44:59,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:01,441 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:01,452 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:03,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:45:03,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:45:03,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:45:04,253 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:04,271 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:04,282 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:04,663 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:45:06,300 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:06,314 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:09,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:09,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:09,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:11,138 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:11,150 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:14,097 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:14,107 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:14,254 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:15,577 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:45:16,126 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:16,135 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:18,061 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:45:18,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:45:18,062 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:45:18,666 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:45:18,966 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:18,975 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:20,235 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:20,996 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:21,007 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:23,878 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:23,892 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:25,234 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:25,913 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:25,927 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:28,728 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:28,739 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:30,672 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:45:30,759 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:30,769 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:31,142 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:33,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:45:33,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:45:33,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:45:34,076 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:45:34,094 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:34,103 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:36,294 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:36,822 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:36,832 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:38,850 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:38,860 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:41,296 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:41,683 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:41,692 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:43,711 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:43,727 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:44,677 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:45:45,578 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:45:46,579 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:46,704 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:46,716 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:48,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:45:48,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:45:48,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:45:48,735 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:48,752 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:51,533 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:51,543 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:52,238 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:53,565 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:53,574 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:56,393 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:56,404 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:57,239 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:45:58,422 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:58,435 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:45:58,681 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:46:01,372 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:01,384 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:02,656 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:03,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:46:03,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:46:03,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:46:03,406 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:03,415 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:06,328 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:06,345 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:08,284 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:08,368 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:08,378 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:11,660 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:46:11,677 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:11,689 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:12,685 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:46:14,264 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:14,421 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:14,432 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:15,580 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:46:16,454 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:16,463 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:18,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:46:18,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:46:18,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:46:19,284 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:19,295 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:20,907 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:21,317 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:21,326 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:24,155 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:24,167 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:25,976 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:26,188 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:26,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:26,689 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:46:29,222 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:29,232 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:30,976 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:31,251 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:31,264 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:33,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:46:33,063 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:46:33,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:46:34,073 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:34,089 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:36,108 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:36,119 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:36,211 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:39,050 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:39,064 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:41,081 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:41,091 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:41,580 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:41,694 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:46:44,059 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:44,069 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:45,581 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:46:46,090 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:46,100 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:46,582 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:48,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:46:48,063 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:46:48,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:46:49,548 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:46:49,566 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:49,575 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:52,156 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:52,520 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:52,530 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:53,697 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:46:54,550 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:54,560 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:57,164 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:46:57,338 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:57,352 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:59,370 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:46:59,379 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:02,164 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:02,165 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:02,179 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:03,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:47:03,063 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:47:03,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:47:04,200 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:04,209 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:07,256 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:07,267 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:07,630 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:07,702 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:09,289 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:09,298 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:12,149 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:12,159 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:12,630 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:14,181 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:14,191 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:15,583 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:47:16,982 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:16,993 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:18,026 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:18,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:47:18,063 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:47:18,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:47:19,014 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:19,026 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:21,707 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:21,856 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:21,870 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:23,421 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:23,888 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:23,900 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:27,209 ERROR gpu :1895319 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 23:47:27,229 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:27,242 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:28,421 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:29,942 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:29,963 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:31,983 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:31,996 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:33,062 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:47:33,063 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:47:33,063 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:47:33,472 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:34,954 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:34,972 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:35,711 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:36,990 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:36,999 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:38,473 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:39,781 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:39,791 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:41,811 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:41,820 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:42,920 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 23:47:42,923 DEBUG SenderThread:1895319 [sender.py:send():382] send: history +2024-03-27 23:47:42,923 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: summary_record +2024-03-27 23:47:42,924 INFO SenderThread:1895319 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 23:47:43,714 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/wandb-summary.json +2024-03-27 23:47:43,714 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:43,924 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:44,702 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:44,712 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:44,714 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:45,715 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log +2024-03-27 23:47:45,735 DEBUG SenderThread:1895319 [sender.py:send():382] send: stats +2024-03-27 23:47:46,734 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:46,748 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:49,663 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:49,737 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 23:47:49,743 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:50,570 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 23:47:50,570 DEBUG HandlerThread:1895319 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 23:47:50,571 DEBUG SenderThread:1895319 [sender.py:send_request():409] send_request: stop_status +2024-03-27 23:47:51,777 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:51,787 ERROR gpu :1895319 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 23:47:53,717 INFO Thread-13 :1895319 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft-frozen-encoder/wandb/run-20240327_190513-7p2x8a0l/files/output.log