diff --git "a/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" "b/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" --- "a/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" +++ "b/wandb/run-20220129_215451-1vipdbow/logs/debug-internal.log" @@ -3097,3 +3097,937 @@ 2022-01-29 22:59:57,300 DEBUG SenderThread:37713 [sender.py:send():234] send: stats 2022-01-29 23:00:02,058 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status 2022-01-29 23:00:02,058 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:00:16,076 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:17,241 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:00:17,241 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:00:22,080 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:27,588 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:00:28,083 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:32,086 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:32,404 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:00:32,404 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:00:36,089 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:40,091 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:46,097 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:47,572 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:00:47,573 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:00:50,099 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:55,102 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:00:57,911 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:00:59,106 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:02,875 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:01:02,876 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:01:03,108 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:07,111 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:11,114 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:15,117 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:18,231 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:01:18,232 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:01:19,120 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:23,122 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:27,125 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:28,116 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:01:29,126 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:33,129 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:33,484 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:01:33,484 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:01:37,132 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:39,133 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:43,136 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:45,138 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:48,738 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:01:48,739 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:01:49,141 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:52,144 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:56,147 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:58,148 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:01:58,346 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:02:02,151 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:04,041 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:02:04,045 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:02:04,152 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:08,154 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:10,155 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:12,157 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:16,159 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:18,161 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:19,283 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:02:19,284 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:02:20,162 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:22,164 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:26,167 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:28,168 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:28,629 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:02:30,169 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:32,170 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:34,172 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:34,563 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:02:34,564 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:02:36,173 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:38,175 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:40,177 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:42,178 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:44,179 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:46,181 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:48,183 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:49,781 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:02:49,782 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:02:50,184 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:52,185 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:58,189 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:02:58,846 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:03:04,193 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:04,998 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:03:04,998 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:03:08,196 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:15,200 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:19,203 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:20,151 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:03:20,152 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:03:23,206 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:27,209 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:29,105 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:03:33,213 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:35,421 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:03:35,422 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:03:37,218 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:41,219 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:43,221 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:47,224 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:50,931 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:03:50,932 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:03:51,227 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:55,230 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:59,233 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:03:59,448 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:04:03,236 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:05,238 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:06,264 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:04:06,265 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:04:09,241 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:13,244 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:16,246 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:20,249 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:21,461 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:04:21,461 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:04:22,251 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:26,254 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:29,708 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:04:30,257 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:32,259 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:36,262 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:36,782 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:04:36,783 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:04:38,264 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:40,265 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:44,269 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:46,270 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:48,272 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:52,034 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:04:52,034 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:04:52,275 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:54,277 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:56,279 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:58,281 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:04:59,923 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:05:02,284 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:04,286 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:06,288 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:07,269 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:05:07,270 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:05:08,289 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:10,291 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:12,293 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:14,295 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:16,296 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:18,298 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:20,300 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:22,302 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:22,494 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:05:22,495 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:05:24,304 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:27,306 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:27,719 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:05:27,746 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:05:27,780 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:05:28,307 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:05:29,308 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:30,212 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:05:35,313 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:37,664 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:05:37,665 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:05:41,317 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:45,320 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:51,323 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:52,834 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:05:52,835 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:05:55,326 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:05:59,328 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:00,544 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:06:03,345 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:08,049 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:06:08,049 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:06:08,348 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:12,351 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:16,354 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:20,357 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:23,445 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:06:23,446 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:06:24,360 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:28,363 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:30,799 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:06:32,367 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:36,374 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:38,376 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:38,789 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:06:38,790 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:06:42,379 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:46,381 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:48,383 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:52,386 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:54,147 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:06:54,148 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:06:56,390 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:06:58,391 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:01,020 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:07:01,393 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:05,396 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:07,397 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:09,486 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:07:09,487 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:07:11,400 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:13,400 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:17,407 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:19,408 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:23,411 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:24,754 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:07:24,755 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:07:25,412 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:27,413 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:31,318 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:07:31,416 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:33,418 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:35,420 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:37,422 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:39,968 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:07:39,968 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:07:41,425 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:43,427 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:45,429 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:47,431 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:49,432 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:51,434 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:53,436 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:55,138 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:07:55,138 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:07:55,438 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:57,440 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:07:59,442 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:01,444 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:01,636 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:08:03,446 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:05,448 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:10,471 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:08:10,472 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:08:11,451 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:17,456 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:22,459 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:25,864 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:08:25,864 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:08:26,461 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:31,942 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:08:32,465 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:36,468 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:40,471 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:41,123 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:08:41,124 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:08:44,473 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:48,476 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:52,479 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:08:56,414 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:08:56,414 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:08:56,481 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:01,485 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:02,223 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:09:05,488 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:09,490 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:11,741 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:09:11,742 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:09:13,493 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:17,496 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:19,498 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:23,500 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:27,043 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:09:27,044 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:09:27,503 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:31,506 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:32,531 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:09:33,507 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:37,510 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:39,512 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:42,286 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:09:42,287 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:09:43,514 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:45,516 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:49,518 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:51,520 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:55,522 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:57,524 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:09:57,619 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:09:57,620 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:10:01,532 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:02,830 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:10:03,533 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:05,534 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:10,538 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:12,539 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:12,934 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:10:12,934 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:10:14,540 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:16,542 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:18,543 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:22,546 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:24,547 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:26,548 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:28,192 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:10:28,193 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:10:28,550 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:30,551 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:32,553 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:33,149 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:10:34,554 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:36,555 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:38,557 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:40,558 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:42,559 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:43,069 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:10:43,195 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:10:43,233 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:10:43,560 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:10:43,627 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:10:43,629 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:10:44,561 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:45,562 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:50,565 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:56,569 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:10:59,075 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:10:59,076 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:11:00,572 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:03,420 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:11:06,576 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:10,579 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:14,354 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:11:14,355 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:11:14,581 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:18,584 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:24,588 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:28,590 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:29,642 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:11:29,642 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:11:32,593 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:33,702 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:11:36,595 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:39,598 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:43,601 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:44,810 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:11:44,811 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:11:47,604 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:51,607 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:55,609 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:57,611 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:11:59,981 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:11:59,981 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:12:01,613 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:03,945 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:12:05,616 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:07,617 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:11,621 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:15,271 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:12:15,271 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:12:15,623 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:17,625 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:21,628 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:23,629 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:27,632 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:29,634 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:30,440 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:12:30,441 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:12:31,636 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:34,287 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:12:35,639 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:37,641 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:39,643 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:44,646 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:45,701 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:12:45,702 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:12:46,648 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:48,649 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:52,652 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:54,653 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:56,654 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:12:58,656 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:00,657 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:00,895 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:13:00,896 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:13:02,659 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:04,624 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:13:04,660 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:06,661 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:08,662 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:10,663 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:12,664 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:14,666 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:16,202 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:13:16,203 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:13:16,667 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:18,668 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:20,670 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:26,674 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:31,372 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:13:31,372 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:13:32,678 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:34,854 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:13:38,682 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:43,686 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:46,587 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:13:46,587 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:13:47,690 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:53,695 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:13:57,698 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:01,701 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:01,825 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:14:01,825 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:14:05,078 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:14:05,704 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:09,707 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:13,710 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:17,032 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:14:17,032 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:14:17,714 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:21,717 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:25,720 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:27,722 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:31,725 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:32,320 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:14:32,321 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:14:35,358 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:14:36,729 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:40,731 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:44,734 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:46,736 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:47,614 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:14:47,614 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:14:50,739 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:52,741 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:56,745 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:14:58,747 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:02,750 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:02,772 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:15:02,772 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:15:04,752 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:05,645 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:15:08,755 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:10,757 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:14,760 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:16,762 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:18,011 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:15:18,011 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:15:18,765 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:22,768 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:24,769 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:26,771 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:28,772 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:32,774 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:33,320 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:15:33,320 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:15:34,776 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:35,940 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:15:36,777 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:38,779 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:40,781 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:42,783 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:44,785 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:46,789 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:48,571 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:15:48,572 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:15:49,792 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:51,793 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:53,796 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:55,799 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:57,801 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:58,498 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:15:58,608 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:15:58,649 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:15:58,802 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:15:58,803 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:15:59,804 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:03,828 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:16:03,830 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:16:06,215 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:16:07,810 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:11,812 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:17,816 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:18,995 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:16:18,996 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:16:21,820 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:25,823 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:31,826 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:34,295 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:16:34,296 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:16:35,828 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:36,562 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:16:39,831 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:43,836 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:47,839 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:49,546 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:16:49,547 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:16:51,842 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:56,845 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:16:58,847 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:02,849 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:04,777 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:17:04,778 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:17:06,841 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:17:06,853 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:10,856 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:12,857 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:16,863 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:20,060 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:17:20,061 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:17:20,866 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:22,868 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:26,872 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:28,874 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:32,877 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:34,879 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:35,312 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:17:35,312 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:17:37,091 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:17:38,882 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:40,883 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:44,886 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:46,887 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:50,556 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:17:50,557 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:17:50,891 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:52,891 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:54,893 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:17:56,894 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:00,897 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:03,898 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:05,709 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:18:05,710 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:18:05,900 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:07,377 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:18:07,902 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:11,904 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:13,905 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:15,906 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:17,907 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:19,909 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:20,931 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:18:20,931 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:18:21,910 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:23,911 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:25,913 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:27,914 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:29,915 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:31,917 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:33,918 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:35,919 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:36,452 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:18:36,452 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:18:37,652 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:18:41,923 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:47,927 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:51,827 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:18:51,827 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:18:51,929 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:18:57,933 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:01,936 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:05,943 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:07,110 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:19:07,110 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:19:07,923 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:19:09,946 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:14,949 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:18,951 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:22,371 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:19:22,371 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:19:22,954 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:26,957 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:30,960 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:34,967 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:37,729 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:19:37,730 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:19:38,222 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:19:38,970 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:42,973 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:46,976 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:48,978 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:52,981 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:53,107 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:19:53,107 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:19:56,984 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:19:58,986 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:02,988 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:06,991 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:08,417 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:20:08,418 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:20:08,604 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:20:08,992 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:12,996 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:14,998 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:20,001 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:22,002 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:23,749 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:20:23,749 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:20:24,003 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:28,006 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:30,008 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:34,011 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:36,013 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:38,015 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:38,703 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:20:38,993 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:20:38,994 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:20:40,017 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:44,020 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:46,022 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:48,023 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:50,025 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:52,027 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:54,028 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:54,247 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:20:54,247 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:20:56,030 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:20:58,031 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:00,033 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:02,034 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:04,037 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:06,039 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:08,040 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:08,946 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:21:09,415 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:21:09,416 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:21:10,042 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:11,555 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:21:11,627 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:21:11,659 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:21:12,046 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:21:12,047 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:13,047 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:20,052 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:24,715 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:21:24,716 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:21:25,055 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:31,059 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:35,064 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:39,067 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:39,213 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:21:39,977 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:21:39,978 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:21:43,070 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:49,074 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:53,077 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:21:55,265 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:21:55,265 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:21:57,080 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:01,084 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:05,086 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:09,089 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:09,465 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:22:10,571 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:22:10,571 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:22:11,090 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:15,093 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:19,096 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:23,100 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:26,068 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:22:26,068 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:22:27,103 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:30,106 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:34,109 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:38,112 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:39,702 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:22:40,114 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:41,308 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:22:41,308 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:22:44,117 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:46,119 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:50,122 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:52,123 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:56,127 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:22:56,476 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:22:56,476 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:22:58,128 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:02,131 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:04,133 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:06,135 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:09,976 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:23:10,139 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:11,680 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:23:11,682 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:23:12,140 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:14,142 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:18,145 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:20,147 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:22,149 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:24,151 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:26,153 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:26,931 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:23:26,932 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:23:30,156 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:32,158 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:35,161 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:37,163 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:39,165 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:40,235 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:23:41,167 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:42,090 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:23:42,091 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:23:43,169 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:45,172 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:47,174 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:49,175 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:51,177 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:57,182 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:23:57,276 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:23:57,277 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:24:03,186 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:07,189 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:10,493 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:24:11,191 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:12,520 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:24:12,521 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:24:17,196 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:21,198 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:25,201 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:27,685 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:24:27,686 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:24:29,204 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:33,207 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:37,211 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:40,790 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:24:41,214 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:43,011 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:24:43,012 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:24:46,218 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:50,221 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:54,224 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:56,226 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:24:58,197 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:24:58,197 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:25:00,229 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:04,232 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:08,235 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:10,237 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:11,072 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:25:13,534 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:25:13,535 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:25:14,240 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:18,243 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:20,245 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:24,247 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:26,249 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:28,251 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:28,761 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:25:28,762 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:25:32,254 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:34,256 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:38,259 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:40,261 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:41,341 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:25:43,927 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:25:43,927 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:25:44,264 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:46,266 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:48,268 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:52,271 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:54,273 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:56,294 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:58,295 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:25:59,101 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:25:59,101 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:26:00,297 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:04,300 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:06,302 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:08,304 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:10,305 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:11,630 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:26:12,307 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:14,259 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:26:14,259 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:26:14,307 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:17,309 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:19,311 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:21,312 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:23,313 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:25,028 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:26:25,093 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:26:25,135 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:26:25,314 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:26:25,314 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:27,315 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:29,317 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:29,532 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:26:29,533 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:26:33,319 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:35,321 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:39,323 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:41,325 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:41,875 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:26:44,704 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:26:44,704 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:26:45,327 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:47,329 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:49,330 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:53,332 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:55,333 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:26:57,334 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:00,000 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:27:00,001 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:27:01,337 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:03,338 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:05,340 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:09,346 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:11,347 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:12,116 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:27:13,348 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:15,442 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:27:15,442 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:27:17,352 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:19,354 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:21,356 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:26,359 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:28,361 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:30,363 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:30,652 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:27:30,653 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:27:34,366 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:36,368 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:38,370 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:42,337 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:27:42,373 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:44,375 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:45,887 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:27:45,888 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:27:46,377 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:50,380 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:52,382 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:54,384 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:27:58,387 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:00,389 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:01,109 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:28:01,110 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:28:02,391 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:06,394 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:08,396 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:10,397 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:12,591 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:28:14,399 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:16,397 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:28:16,398 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:28:17,405 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:19,406 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:23,409 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:25,410 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:29,412 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:31,414 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:31,624 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:28:31,625 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:28:33,415 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:37,419 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:39,421 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:41,423 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:42,840 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:28:45,426 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:46,829 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:28:46,830 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:28:47,428 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:49,430 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:51,432 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:55,435 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:28:57,437 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:01,440 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:02,152 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:29:02,152 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:29:03,441 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:05,443 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:09,446 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:11,448 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:13,067 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:29:13,450 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:17,399 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:29:17,400 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:29:17,453 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:20,455 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:22,457 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:26,460 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:28,461 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:32,464 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:32,632 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:29:32,632 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:29:34,465 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:36,466 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:40,469 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:42,470 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:43,316 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:29:46,473 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:47,970 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:29:47,971 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:29:48,474 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:50,476 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:54,478 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:29:56,479 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:00,481 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:02,482 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:03,202 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:30:03,202 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:30:04,484 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:08,486 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:10,487 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:12,489 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:13,535 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:30:16,491 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:18,419 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:30:18,420 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:30:19,493 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:23,495 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:25,496 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:27,498 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:31,500 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:33,502 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:33,635 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:30:33,636 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:30:35,503 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:37,505 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:30:43,764 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:30:48,829 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:30:48,830 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:31:05,297 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:31:05,298 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:31:14,009 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:31:20,504 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:31:20,505 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:31:22,179 DEBUG SenderThread:37713 [sender.py:send():234] send: history +2022-01-29 23:31:22,214 DEBUG SenderThread:37713 [sender.py:send():234] send: summary +2022-01-29 23:31:22,252 INFO SenderThread:37713 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 23:31:22,528 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/wandb-summary.json +2022-01-29 23:31:23,530 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:31:24,531 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:31:25,533 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:31:35,539 INFO Thread-8 :37713 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_215451-1vipdbow/files/output.log +2022-01-29 23:31:35,697 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:31:35,698 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:31:44,251 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:31:50,886 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:31:50,886 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:32:06,054 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:32:06,055 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:32:14,490 DEBUG SenderThread:37713 [sender.py:send():234] send: stats +2022-01-29 23:32:21,220 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:32:21,221 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:32:36,390 DEBUG HandlerThread:37713 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 23:32:36,391 DEBUG SenderThread:37713 [sender.py:send_request():248] send_request: stop_status +2022-01-29 23:32:44,724 DEBUG SenderThread:37713 [sender.py:send():234] send: stats